diff --git a/test/packages/apache/_dev/build/build.yml b/test/packages/apache/_dev/build/build.yml new file mode 100644 index 0000000000..08d85edcf9 --- /dev/null +++ b/test/packages/apache/_dev/build/build.yml @@ -0,0 +1,3 @@ +dependencies: + ecs: + reference: git@1.12 diff --git a/test/packages/apache/_dev/build/docs/README.md b/test/packages/apache/_dev/build/docs/README.md index 61402727a9..f57e928c13 100644 --- a/test/packages/apache/_dev/build/docs/README.md +++ b/test/packages/apache/_dev/build/docs/README.md @@ -5,8 +5,8 @@ logs created by the Apache server. ## Compatibility -The Apache datasets were tested with Apache 2.4.12 and 2.4.20 and are expected to work with -all versions >= 2.2.31 and >= 2.4.16. +The Apache datasets were tested with Apache 2.4.12 and 2.4.46 and are expected to work with +all versions >= 2.2.31 and >= 2.4.16 (independent from operating system). ## Logs @@ -31,4 +31,4 @@ generated by the `mod_status` module. {{event "status"}} -{{fields "status"}} \ No newline at end of file +{{fields "status"}} diff --git a/test/packages/apache/_dev/deploy/docker/Dockerfile b/test/packages/apache/_dev/deploy/docker/Dockerfile index 16a87baed7..d6a2916a3c 100644 --- a/test/packages/apache/_dev/deploy/docker/Dockerfile +++ b/test/packages/apache/_dev/deploy/docker/Dockerfile @@ -1,4 +1,4 @@ -ARG SERVICE_VERSION=${SERVICE_VERSION:-2.4.20} +ARG SERVICE_VERSION=${SERVICE_VERSION:-2.4.46} FROM httpd:$SERVICE_VERSION RUN sed -i "/jessie-updates/d" /etc/apt/sources.list RUN apt-get update && apt-get install -y curl diff --git a/test/packages/apache/_dev/deploy/docker/docker-compose.yml b/test/packages/apache/_dev/deploy/docker/docker-compose.yml index 0c20270dd2..6feb176f98 100644 --- a/test/packages/apache/_dev/deploy/docker/docker-compose.yml +++ b/test/packages/apache/_dev/deploy/docker/docker-compose.yml @@ -4,10 +4,7 @@ services: # Commented out `image:` below until we have a process to refresh the hosted images from # Dockerfiles in this repo. Until then, we build the image locally using `build:` below. # image: docker.elastic.co/integrations-ci/beats-apache:${SERVICE_VERSION:-2.4.20}-1 - build: - context: . - args: - SERVICE_VERSION: ${SERVICE_VERSION} + build: . ports: - 80 volumes: diff --git a/test/packages/apache/_dev/deploy/docker/httpd.conf b/test/packages/apache/_dev/deploy/docker/httpd.conf index f801678890..f402947317 100644 --- a/test/packages/apache/_dev/deploy/docker/httpd.conf +++ b/test/packages/apache/_dev/deploy/docker/httpd.conf @@ -149,6 +149,7 @@ LoadModule dir_module modules/mod_dir.so #LoadModule userdir_module modules/mod_userdir.so LoadModule alias_module modules/mod_alias.so #LoadModule rewrite_module modules/mod_rewrite.so +LoadModule mpm_event_module modules/mod_mpm_event.so # @@ -507,4 +508,4 @@ SSLRandomSeed connect builtin - \ No newline at end of file + diff --git a/test/packages/apache/_dev/deploy/variants.yml b/test/packages/apache/_dev/deploy/variants.yml index d4a4d65d6f..a4f638620c 100644 --- a/test/packages/apache/_dev/deploy/variants.yml +++ b/test/packages/apache/_dev/deploy/variants.yml @@ -1,6 +1,4 @@ variants: - v2420: - SERVICE_VERSION: 2.4.20 - v2423: - SERVICE_VERSION: 2.4.23 -default: v2420 + v2: + SERVICE_VERSION: 2.4.46 +default: v2 diff --git a/test/packages/apache/changelog.yml b/test/packages/apache/changelog.yml index 49d5938bf6..89cff330bc 100644 --- a/test/packages/apache/changelog.yml +++ b/test/packages/apache/changelog.yml @@ -1,6 +1,111 @@ # newer versions go on top -- version: "0.0.1" +- version: "999.999.999" + changes: + - description: Change test public IPs to the supported subset + type: bugfix + link: https://github.com/elastic/integrations/pull/2327 +- version: "1.3.2" + changes: + - description: Fix ML module manifest query to ignore frozen and cold tiers + type: bugfix + link: https://github.com/elastic/integrations/pull/2217 +- version: "1.3.1" + changes: + - description: Fix parsing of trace log levels + type: bugfix + link: https://github.com/elastic/integrations/pull/2064 +- version: "1.3.0" + changes: + - description: Support Kibana 8.0 + type: enhancement + link: https://github.com/elastic/integrations/pull/2122 +- version: "1.2.0" + changes: + - description: Uniform with guidelines + type: enhancement + link: https://github.com/elastic/integrations/pull/2001 +- version: "1.1.1" + changes: + - description: Fix logic that checks for the 'forwarded' tag + type: bugfix + link: https://github.com/elastic/integrations/pull/1794 +- version: "1.1.0" + changes: + - description: Update to ECS 1.12.0 + type: enhancement + link: https://github.com/elastic/integrations/pull/1686 +- version: "1.0.0" + changes: + - description: Release Apache as GA + type: enhancement + link: https://github.com/elastic/integrations/pull/1607 +- version: "0.9.2" + changes: + - description: Convert to generated ECS fields + type: enhancement + link: https://github.com/elastic/integrations/pull/1463 +- version: '0.9.1' + changes: + - description: update to ECS 1.11.0 + type: enhancement + link: https://github.com/elastic/integrations/pull/1369 +- version: "0.9.0" + changes: + - description: Update integration description + type: enhancement + link: https://github.com/elastic/integrations/pull/1364 +- version: "0.8.1" + changes: + - description: Add support for Splunk authorization tokens + type: enhancement + link: https://github.com/elastic/integrations/pull/1147 +- version: "0.8.0" + changes: + - description: Set event.module and event.dataset + type: bugfix + link: https://github.com/elastic/integrations/pull/1230 +- version: "0.7.1" + changes: + - description: Fix bug in Third Party REST API ingest pipeline + type: bugfix + link: https://github.com/elastic/integrations/pull/1201 +- version: "0.7.0" + changes: + - description: Update to ECS 1.10.0 and adding items that all packages should have + type: enhancement + link: https://github.com/elastic/integrations/pull/1068 +- version: "0.6.0" + changes: + - description: Render units and metric types in exported fields table + type: enhancement + link: https://github.com/elastic/integrations/pull/1028 +- version: "0.5.1" + changes: + - description: Move ecs.version to the ingest pipeline and make event.original optional + type: enhancement + link: https://github.com/elastic/integrations/pull/1025 +- version: "0.5.0" + changes: + - description: Adds ML jobs for finding unusual activity in HTTP access logs + type: enhancement + link: https://github.com/elastic/integrations/pull/910 +- version: "0.4.1" + changes: + - description: update to ECS 1.9.0 + type: enhancement + link: https://github.com/elastic/integrations/pull/832 +- version: "0.3.5" + changes: + - description: Updating package owner + type: enhancement + link: https://github.com/elastic/integrations/pull/766 +- version: "0.3.4" + changes: + - description: Use correct types for `source.port` and `source.ip` + type: bugfix + link: https://github.com/elastic/integrations/pull/737 +- version: "0.1.0" changes: - description: initial release type: enhancement # can be one of: enhancement, bugfix, breaking-change - link: https://github.com/elastic/elastic-package/pull/109 + link: https://github.com/elastic/integrations/pull/98 diff --git a/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log b/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log new file mode 100644 index 0000000000..4e2cbbe7e5 --- /dev/null +++ b/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log @@ -0,0 +1,7 @@ +::1 - - [26/Dec/2016:16:16:29 +0200] "GET /favicon.ico HTTP/1.1" 404 209 +192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] "GET /hello HTTP/1.1" 404 499 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0" +::1 - - [26/Dec/2016:16:16:48 +0200] "-" 408 - +172.17.0.1 - - [29/May/2017:19:02:48 +0000] "GET /stringpatch HTTP/1.1" 404 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-" +monitoring-server - - [29/May/2017:19:02:48 +0000] "GET /status HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-" +127.0.0.1 - - [02/Feb/2019:05:38:45 +0100] "-" 408 152 "-" "-" +monitoring-server - - [29/May/2017:19:02:48 +0000] "GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-" \ No newline at end of file diff --git a/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json b/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json new file mode 100644 index 0000000000..481ed5c4e5 --- /dev/null +++ b/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json @@ -0,0 +1,354 @@ +{ + "expected": [ + { + "apache": { + "access": {} + }, + "@timestamp": "2016-12-26T14:16:29.000Z", + "ecs": { + "version": "1.12.0" + }, + "http": { + "request": { + "method": "GET" + }, + "version": "1.1", + "response": { + "body": { + "bytes": 209 + }, + "status_code": 404 + } + }, + "source": { + "address": "::1", + "ip": "::1" + }, + "event": { + "ingested": "2021-12-09T13:30:29.903774500Z", + "original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209", + "category": "web", + "kind": "event", + "created": "2020-04-28T11:07:58.223Z", + "outcome": "failure" + }, + "user": { + "name": "-" + }, + "url": { + "path": "/favicon.ico", + "extension": "ico", + "original": "/favicon.ico" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, + "url": { + "path": "/hello", + "original": "/hello" + }, + "tags": [ + "preserve_original_event" + ], + "apache": { + "access": {} + }, + "@timestamp": "2016-12-26T16:22:13.000Z", + "ecs": { + "version": "1.12.0" + }, + "http": { + "request": { + "method": "GET", + "referrer": "-" + }, + "version": "1.1", + "response": { + "body": { + "bytes": 499 + }, + "status_code": 404 + } + }, + "event": { + "ingested": "2021-12-09T13:30:29.903783200Z", + "original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", + "category": "web", + "kind": "event", + "created": "2020-04-28T11:07:58.223Z", + "outcome": "failure" + }, + "user": { + "name": "-" + }, + "user_agent": { + "name": "Firefox", + "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0", + "os": { + "name": "Mac OS X", + "version": "10.12", + "full": "Mac OS X 10.12" + }, + "device": { + "name": "Mac" + }, + "version": "50.0." + } + }, + { + "apache": { + "access": {} + }, + "@timestamp": "2016-12-26T14:16:48.000Z", + "ecs": { + "version": "1.12.0" + }, + "http": { + "response": { + "status_code": 408 + } + }, + "source": { + "address": "::1", + "ip": "::1" + }, + "event": { + "ingested": "2021-12-09T13:30:29.903788600Z", + "original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -", + "category": "web", + "kind": "event", + "created": "2020-04-28T11:07:58.223Z", + "outcome": "failure" + }, + "user": { + "name": "-" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "source": { + "address": "172.17.0.1", + "ip": "172.17.0.1" + }, + "url": { + "path": "/stringpatch", + "original": "/stringpatch" + }, + "tags": [ + "preserve_original_event" + ], + "apache": { + "access": {} + }, + "@timestamp": "2017-05-29T19:02:48.000Z", + "ecs": { + "version": "1.12.0" + }, + "http": { + "request": { + "method": "GET", + "referrer": "-" + }, + "version": "1.1", + "response": { + "body": { + "bytes": 612 + }, + "status_code": 404 + } + }, + "event": { + "ingested": "2021-12-09T13:30:29.903792500Z", + "original": "172.17.0.1 - - [29/May/2017:19:02:48 +0000] \"GET /stringpatch HTTP/1.1\" 404 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", + "category": "web", + "kind": "event", + "created": "2020-04-28T11:07:58.223Z", + "outcome": "failure" + }, + "user": { + "name": "-" + }, + "user_agent": { + "name": "Firefox Alpha", + "original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", + "os": { + "name": "Windows", + "version": "7", + "full": "Windows 7" + }, + "device": { + "name": "Other" + }, + "version": "15.0.a2" + } + }, + { + "source": { + "address": "monitoring-server", + "domain": "monitoring-server" + }, + "url": { + "path": "/status", + "original": "/status" + }, + "tags": [ + "preserve_original_event" + ], + "apache": { + "access": {} + }, + "@timestamp": "2017-05-29T19:02:48.000Z", + "ecs": { + "version": "1.12.0" + }, + "http": { + "request": { + "method": "GET", + "referrer": "-" + }, + "version": "1.1", + "response": { + "body": { + "bytes": 612 + }, + "status_code": 200 + } + }, + "event": { + "ingested": "2021-12-09T13:30:29.903797600Z", + "original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /status HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", + "category": "web", + "kind": "event", + "created": "2020-04-28T11:07:58.223Z", + "outcome": "success" + }, + "user": { + "name": "-" + }, + "user_agent": { + "name": "Firefox Alpha", + "original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", + "os": { + "name": "Windows", + "version": "7", + "full": "Windows 7" + }, + "device": { + "name": "Other" + }, + "version": "15.0.a2" + } + }, + { + "apache": { + "access": {} + }, + "@timestamp": "2019-02-02T04:38:45.000Z", + "ecs": { + "version": "1.12.0" + }, + "http": { + "request": { + "referrer": "-" + }, + "response": { + "body": { + "bytes": 152 + }, + "status_code": 408 + } + }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2021-12-09T13:30:29.903803900Z", + "original": "127.0.0.1 - - [02/Feb/2019:05:38:45 +0100] \"-\" 408 152 \"-\" \"-\"", + "category": "web", + "kind": "event", + "created": "2020-04-28T11:07:58.223Z", + "outcome": "failure" + }, + "user": { + "name": "-" + }, + "user_agent": { + "name": "Other", + "device": { + "name": "Other" + }, + "original": "-" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "source": { + "address": "monitoring-server", + "domain": "monitoring-server" + }, + "url": { + "path": "/A Beka G1 Howe/029_AND_30/15 reading elephants.mp4", + "extension": "mp4", + "original": "/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4" + }, + "tags": [ + "preserve_original_event" + ], + "apache": { + "access": {} + }, + "@timestamp": "2017-05-29T19:02:48.000Z", + "ecs": { + "version": "1.12.0" + }, + "http": { + "request": { + "method": "GET", + "referrer": "-" + }, + "version": "1.1", + "response": { + "body": { + "bytes": 612 + }, + "status_code": 200 + } + }, + "event": { + "ingested": "2021-12-09T13:30:29.903809300Z", + "original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", + "category": "web", + "kind": "event", + "created": "2020-04-28T11:07:58.223Z", + "outcome": "success" + }, + "user": { + "name": "-" + }, + "user_agent": { + "name": "Firefox Alpha", + "original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", + "os": { + "name": "Windows", + "version": "7", + "full": "Windows 7" + }, + "device": { + "name": "Other" + }, + "version": "15.0.a2" + } + } + ] +} \ No newline at end of file diff --git a/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log b/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log new file mode 100644 index 0000000000..6b1ba50b17 --- /dev/null +++ b/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log @@ -0,0 +1,6 @@ +::1 - - [26/Dec/2016:16:16:28 +0200] "GET / HTTP/1.1" 200 45 +::1 - - [26/Dec/2016:16:16:29 +0200] "GET /favicon.ico HTTP/1.1" 404 209 +::1 - - [26/Dec/2016:16:16:48 +0200] "-" 408 - +89.160.20.156 - - [26/Dec/2016:18:23:35 +0200] "GET / HTTP/1.1" 200 45 +89.160.20.156 - - [26/Dec/2016:18:23:41 +0200] "GET /notfound HTTP/1.1" 404 206 +89.160.20.156 - - [26/Dec/2016:18:23:45 +0200] "GET /hmm HTTP/1.1" 404 201 diff --git a/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json b/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json new file mode 100644 index 0000000000..c893102f46 --- /dev/null +++ b/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json @@ -0,0 +1,306 @@ +{ + "expected": [ + { + "apache": { + "access": {} + }, + "@timestamp": "2016-12-26T14:16:28.000Z", + "ecs": { + "version": "1.12.0" + }, + "http": { + "request": { + "method": "GET" + }, + "version": "1.1", + "response": { + "body": { + "bytes": 45 + }, + "status_code": 200 + } + }, + "source": { + "address": "::1", + "ip": "::1" + }, + "event": { + "ingested": "2021-12-09T13:30:30.879403900Z", + "original": "::1 - - [26/Dec/2016:16:16:28 +0200] \"GET / HTTP/1.1\" 200 45", + "category": "web", + "kind": "event", + "created": "2020-04-28T11:07:58.223Z", + "outcome": "success" + }, + "user": { + "name": "-" + }, + "url": { + "path": "/", + "original": "/" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "apache": { + "access": {} + }, + "@timestamp": "2016-12-26T14:16:29.000Z", + "ecs": { + "version": "1.12.0" + }, + "http": { + "request": { + "method": "GET" + }, + "version": "1.1", + "response": { + "body": { + "bytes": 209 + }, + "status_code": 404 + } + }, + "source": { + "address": "::1", + "ip": "::1" + }, + "event": { + "ingested": "2021-12-09T13:30:30.879409400Z", + "original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209", + "category": "web", + "kind": "event", + "created": "2020-04-28T11:07:58.223Z", + "outcome": "failure" + }, + "user": { + "name": "-" + }, + "url": { + "path": "/favicon.ico", + "extension": "ico", + "original": "/favicon.ico" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "apache": { + "access": {} + }, + "@timestamp": "2016-12-26T14:16:48.000Z", + "ecs": { + "version": "1.12.0" + }, + "http": { + "response": { + "status_code": 408 + } + }, + "source": { + "address": "::1", + "ip": "::1" + }, + "event": { + "ingested": "2021-12-09T13:30:30.879413800Z", + "original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -", + "category": "web", + "kind": "event", + "created": "2020-04-28T11:07:58.223Z", + "outcome": "failure" + }, + "user": { + "name": "-" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "apache": { + "access": {} + }, + "@timestamp": "2016-12-26T16:23:35.000Z", + "ecs": { + "version": "1.12.0" + }, + "http": { + "request": { + "method": "GET" + }, + "version": "1.1", + "response": { + "body": { + "bytes": 45 + }, + "status_code": 200 + } + }, + "source": { + "geo": { + "continent_name": "Europe", + "region_iso_code": "SE-AB", + "city_name": "Tumba", + "country_iso_code": "SE", + "country_name": "Sweden", + "region_name": "Stockholm", + "location": { + "lon": 17.8167, + "lat": 59.2 + } + }, + "as": { + "number": 29518, + "organization": { + "name": "Bredband2 AB" + } + }, + "address": "89.160.20.156", + "ip": "89.160.20.156" + }, + "event": { + "ingested": "2021-12-09T13:30:30.879418Z", + "original": "89.160.20.156 - - [26/Dec/2016:18:23:35 +0200] \"GET / HTTP/1.1\" 200 45", + "category": "web", + "kind": "event", + "created": "2020-04-28T11:07:58.223Z", + "outcome": "success" + }, + "user": { + "name": "-" + }, + "url": { + "path": "/", + "original": "/" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "apache": { + "access": {} + }, + "@timestamp": "2016-12-26T16:23:41.000Z", + "ecs": { + "version": "1.12.0" + }, + "http": { + "request": { + "method": "GET" + }, + "version": "1.1", + "response": { + "body": { + "bytes": 206 + }, + "status_code": 404 + } + }, + "source": { + "geo": { + "continent_name": "Europe", + "region_iso_code": "SE-AB", + "city_name": "Tumba", + "country_iso_code": "SE", + "country_name": "Sweden", + "region_name": "Stockholm", + "location": { + "lon": 17.8167, + "lat": 59.2 + } + }, + "as": { + "number": 29518, + "organization": { + "name": "Bredband2 AB" + } + }, + "address": "89.160.20.156", + "ip": "89.160.20.156" + }, + "event": { + "ingested": "2021-12-09T13:30:30.879422100Z", + "original": "89.160.20.156 - - [26/Dec/2016:18:23:41 +0200] \"GET /notfound HTTP/1.1\" 404 206", + "category": "web", + "kind": "event", + "created": "2020-04-28T11:07:58.223Z", + "outcome": "failure" + }, + "user": { + "name": "-" + }, + "url": { + "path": "/notfound", + "original": "/notfound" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "apache": { + "access": {} + }, + "@timestamp": "2016-12-26T16:23:45.000Z", + "ecs": { + "version": "1.12.0" + }, + "http": { + "request": { + "method": "GET" + }, + "version": "1.1", + "response": { + "body": { + "bytes": 201 + }, + "status_code": 404 + } + }, + "source": { + "geo": { + "continent_name": "Europe", + "region_iso_code": "SE-AB", + "city_name": "Tumba", + "country_iso_code": "SE", + "country_name": "Sweden", + "region_name": "Stockholm", + "location": { + "lon": 17.8167, + "lat": 59.2 + } + }, + "as": { + "number": 29518, + "organization": { + "name": "Bredband2 AB" + } + }, + "address": "89.160.20.156", + "ip": "89.160.20.156" + }, + "event": { + "ingested": "2021-12-09T13:30:30.879427100Z", + "original": "89.160.20.156 - - [26/Dec/2016:18:23:45 +0200] \"GET /hmm HTTP/1.1\" 404 201", + "category": "web", + "kind": "event", + "created": "2020-04-28T11:07:58.223Z", + "outcome": "failure" + }, + "user": { + "name": "-" + }, + "url": { + "path": "/hmm", + "original": "/hmm" + }, + "tags": [ + "preserve_original_event" + ] + } + ] +} \ No newline at end of file diff --git a/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log b/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log new file mode 100644 index 0000000000..0a59aed766 --- /dev/null +++ b/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log @@ -0,0 +1,2 @@ +[10/Aug/2018:09:45:56 +0200] 172.30.0.119 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 "GET /nagiosxi/ajaxhelper.php?cmd=getxicoreajax&opts=%7B%22func%22%3A%22get_admin_tasks_html%22%2C%22args%22%3A%22%22%7D&nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21 HTTP/1.1" 1375 +[16/Oct/2019:11:53:47 +0200] 89.160.20.156 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 "GET /appl/ajaxhelper.php?cmd=getxicoreajax&opts=%7B%22func%22%3A%22get_pagetop_alert_content_html%22%2C%22args%22%3A%22%22%7D&nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d HTTP/1.1" - diff --git a/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json b/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json new file mode 100644 index 0000000000..0964c3ba0d --- /dev/null +++ b/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json @@ -0,0 +1,117 @@ +{ + "expected": [ + { + "apache": { + "access": { + "ssl": { + "cipher": "ECDHE-RSA-AES128-GCM-SHA256", + "protocol": "TLSv1.2" + } + } + }, + "@timestamp": "2018-08-10T07:45:56.000Z", + "ecs": { + "version": "1.12.0" + }, + "http": { + "request": { + "method": "GET" + }, + "version": "1.1", + "response": { + "body": { + "bytes": 1375 + } + } + }, + "tls": { + "cipher": "ECDHE-RSA-AES128-GCM-SHA256", + "version": "1.2", + "version_protocol": "tls" + }, + "source": { + "address": "172.30.0.119", + "ip": "172.30.0.119" + }, + "event": { + "ingested": "2021-12-09T13:30:31.533065900Z", + "original": "[10/Aug/2018:09:45:56 +0200] 172.30.0.119 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /nagiosxi/ajaxhelper.php?cmd=getxicoreajax\u0026amp;opts=%7B%22func%22%3A%22get_admin_tasks_html%22%2C%22args%22%3A%22%22%7D\u0026amp;nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21 HTTP/1.1\" 1375", + "category": "web", + "kind": "event", + "created": "2020-04-28T11:07:58.223Z" + }, + "url": { + "path": "/nagiosxi/ajaxhelper.php", + "extension": "php", + "original": "/nagiosxi/ajaxhelper.php?cmd=getxicoreajax\u0026amp;opts=%7B%22func%22%3A%22get_admin_tasks_html%22%2C%22args%22%3A%22%22%7D\u0026amp;nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21", + "query": "cmd=getxicoreajax\u0026amp;opts={\"func\":\"get_admin_tasks_html\",\"args\":\"\"}\u0026amp;nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "apache": { + "access": { + "ssl": { + "cipher": "ECDHE-RSA-AES128-GCM-SHA256", + "protocol": "TLSv1.2" + } + } + }, + "@timestamp": "2019-10-16T09:53:47.000Z", + "ecs": { + "version": "1.12.0" + }, + "http": { + "request": { + "method": "GET" + }, + "version": "1.1" + }, + "tls": { + "cipher": "ECDHE-RSA-AES128-GCM-SHA256", + "version": "1.2", + "version_protocol": "tls" + }, + "source": { + "geo": { + "continent_name": "Europe", + "region_iso_code": "SE-AB", + "city_name": "Tumba", + "country_iso_code": "SE", + "country_name": "Sweden", + "region_name": "Stockholm", + "location": { + "lon": 17.8167, + "lat": 59.2 + } + }, + "as": { + "number": 29518, + "organization": { + "name": "Bredband2 AB" + } + }, + "address": "89.160.20.156", + "ip": "89.160.20.156" + }, + "event": { + "ingested": "2021-12-09T13:30:31.533074100Z", + "original": "[16/Oct/2019:11:53:47 +0200] 89.160.20.156 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /appl/ajaxhelper.php?cmd=getxicoreajax\u0026opts=%7B%22func%22%3A%22get_pagetop_alert_content_html%22%2C%22args%22%3A%22%22%7D\u0026nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d HTTP/1.1\" -", + "category": "web", + "kind": "event", + "created": "2020-04-28T11:07:58.223Z" + }, + "url": { + "path": "/appl/ajaxhelper.php", + "extension": "php", + "original": "/appl/ajaxhelper.php?cmd=getxicoreajax\u0026opts=%7B%22func%22%3A%22get_pagetop_alert_content_html%22%2C%22args%22%3A%22%22%7D\u0026nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d", + "query": "cmd=getxicoreajax\u0026opts={\"func\":\"get_pagetop_alert_content_html\",\"args\":\"\"}\u0026nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d" + }, + "tags": [ + "preserve_original_event" + ] + } + ] +} \ No newline at end of file diff --git a/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log b/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log new file mode 100644 index 0000000000..92d1bdd85a --- /dev/null +++ b/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log @@ -0,0 +1,9 @@ +127.0.0.1 - - [26/Dec/2016:16:18:09 +0000] "GET / HTTP/1.1" 200 491 "-" "Wget/1.13.4 (linux-gnu)" +192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] "GET / HTTP/1.1" 200 484 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36" +192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] "GET /favicon.ico HTTP/1.1" 404 504 "http://192.168.33.72/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36" +192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] "GET / HTTP/1.1" 200 484 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0" +192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] "GET /favicon.ico HTTP/1.1" 404 504 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0" +192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] "GET /favicon.ico HTTP/1.1" 404 504 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0" +192.168.33.1 - - [26/Dec/2016:16:22:10 +0000] "GET /test HTTP/1.1" 404 498 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0" +192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] "GET /hello HTTP/1.1" 404 499 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0" +192.168.33.1 - - [26/Dec/2016:16:22:17 +0000] "GET /crap HTTP/1.1" 404 499 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0" diff --git a/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json b/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json new file mode 100644 index 0000000000..92c297c4b3 --- /dev/null +++ b/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json @@ -0,0 +1,518 @@ +{ + "expected": [ + { + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "url": { + "path": "/", + "original": "/" + }, + "tags": [ + "preserve_original_event" + ], + "apache": { + "access": {} + }, + "@timestamp": "2016-12-26T16:18:09.000Z", + "ecs": { + "version": "1.12.0" + }, + "http": { + "request": { + "method": "GET", + "referrer": "-" + }, + "version": "1.1", + "response": { + "body": { + "bytes": 491 + }, + "status_code": 200 + } + }, + "event": { + "ingested": "2021-12-09T13:30:31.835525800Z", + "original": "127.0.0.1 - - [26/Dec/2016:16:18:09 +0000] \"GET / HTTP/1.1\" 200 491 \"-\" \"Wget/1.13.4 (linux-gnu)\"", + "category": "web", + "kind": "event", + "created": "2020-04-28T11:07:58.223Z", + "outcome": "success" + }, + "user": { + "name": "-" + }, + "user_agent": { + "name": "Wget", + "original": "Wget/1.13.4 (linux-gnu)", + "os": { + "name": "Linux" + }, + "device": { + "name": "Other" + }, + "version": "1.13.4" + } + }, + { + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, + "url": { + "path": "/", + "original": "/" + }, + "tags": [ + "preserve_original_event" + ], + "apache": { + "access": {} + }, + "@timestamp": "2016-12-26T16:22:00.000Z", + "ecs": { + "version": "1.12.0" + }, + "http": { + "request": { + "method": "GET", + "referrer": "-" + }, + "version": "1.1", + "response": { + "body": { + "bytes": 484 + }, + "status_code": 200 + } + }, + "event": { + "ingested": "2021-12-09T13:30:31.835534600Z", + "original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", + "category": "web", + "kind": "event", + "created": "2020-04-28T11:07:58.223Z", + "outcome": "success" + }, + "user": { + "name": "-" + }, + "user_agent": { + "name": "Chrome", + "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36", + "os": { + "name": "Mac OS X", + "version": "10.12.0", + "full": "Mac OS X 10.12.0" + }, + "device": { + "name": "Mac" + }, + "version": "54.0.2840.98" + } + }, + { + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, + "url": { + "path": "/favicon.ico", + "extension": "ico", + "original": "/favicon.ico" + }, + "tags": [ + "preserve_original_event" + ], + "apache": { + "access": {} + }, + "@timestamp": "2016-12-26T16:22:00.000Z", + "ecs": { + "version": "1.12.0" + }, + "http": { + "request": { + "method": "GET", + "referrer": "http://192.168.33.72/" + }, + "version": "1.1", + "response": { + "body": { + "bytes": 504 + }, + "status_code": 404 + } + }, + "event": { + "ingested": "2021-12-09T13:30:31.835540100Z", + "original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"http://192.168.33.72/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", + "category": "web", + "kind": "event", + "created": "2020-04-28T11:07:58.223Z", + "outcome": "failure" + }, + "user": { + "name": "-" + }, + "user_agent": { + "name": "Chrome", + "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36", + "os": { + "name": "Mac OS X", + "version": "10.12.0", + "full": "Mac OS X 10.12.0" + }, + "device": { + "name": "Mac" + }, + "version": "54.0.2840.98" + } + }, + { + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, + "url": { + "path": "/", + "original": "/" + }, + "tags": [ + "preserve_original_event" + ], + "apache": { + "access": {} + }, + "@timestamp": "2016-12-26T16:22:08.000Z", + "ecs": { + "version": "1.12.0" + }, + "http": { + "request": { + "method": "GET", + "referrer": "-" + }, + "version": "1.1", + "response": { + "body": { + "bytes": 484 + }, + "status_code": 200 + } + }, + "event": { + "ingested": "2021-12-09T13:30:31.835543600Z", + "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", + "category": "web", + "kind": "event", + "created": "2020-04-28T11:07:58.223Z", + "outcome": "success" + }, + "user": { + "name": "-" + }, + "user_agent": { + "name": "Firefox", + "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0", + "os": { + "name": "Mac OS X", + "version": "10.12", + "full": "Mac OS X 10.12" + }, + "device": { + "name": "Mac" + }, + "version": "50.0." + } + }, + { + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, + "url": { + "path": "/favicon.ico", + "extension": "ico", + "original": "/favicon.ico" + }, + "tags": [ + "preserve_original_event" + ], + "apache": { + "access": {} + }, + "@timestamp": "2016-12-26T16:22:08.000Z", + "ecs": { + "version": "1.12.0" + }, + "http": { + "request": { + "method": "GET", + "referrer": "-" + }, + "version": "1.1", + "response": { + "body": { + "bytes": 504 + }, + "status_code": 404 + } + }, + "event": { + "ingested": "2021-12-09T13:30:31.835548Z", + "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", + "category": "web", + "kind": "event", + "created": "2020-04-28T11:07:58.223Z", + "outcome": "failure" + }, + "user": { + "name": "-" + }, + "user_agent": { + "name": "Firefox", + "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0", + "os": { + "name": "Mac OS X", + "version": "10.12", + "full": "Mac OS X 10.12" + }, + "device": { + "name": "Mac" + }, + "version": "50.0." + } + }, + { + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, + "url": { + "path": "/favicon.ico", + "extension": "ico", + "original": "/favicon.ico" + }, + "tags": [ + "preserve_original_event" + ], + "apache": { + "access": {} + }, + "@timestamp": "2016-12-26T16:22:08.000Z", + "ecs": { + "version": "1.12.0" + }, + "http": { + "request": { + "method": "GET", + "referrer": "-" + }, + "version": "1.1", + "response": { + "body": { + "bytes": 504 + }, + "status_code": 404 + } + }, + "event": { + "ingested": "2021-12-09T13:30:31.835553700Z", + "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", + "category": "web", + "kind": "event", + "created": "2020-04-28T11:07:58.223Z", + "outcome": "failure" + }, + "user": { + "name": "-" + }, + "user_agent": { + "name": "Firefox", + "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0", + "os": { + "name": "Mac OS X", + "version": "10.12", + "full": "Mac OS X 10.12" + }, + "device": { + "name": "Mac" + }, + "version": "50.0." + } + }, + { + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, + "url": { + "path": "/test", + "original": "/test" + }, + "tags": [ + "preserve_original_event" + ], + "apache": { + "access": {} + }, + "@timestamp": "2016-12-26T16:22:10.000Z", + "ecs": { + "version": "1.12.0" + }, + "http": { + "request": { + "method": "GET", + "referrer": "-" + }, + "version": "1.1", + "response": { + "body": { + "bytes": 498 + }, + "status_code": 404 + } + }, + "event": { + "ingested": "2021-12-09T13:30:31.835559600Z", + "original": "192.168.33.1 - - [26/Dec/2016:16:22:10 +0000] \"GET /test HTTP/1.1\" 404 498 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", + "category": "web", + "kind": "event", + "created": "2020-04-28T11:07:58.223Z", + "outcome": "failure" + }, + "user": { + "name": "-" + }, + "user_agent": { + "name": "Firefox", + "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0", + "os": { + "name": "Mac OS X", + "version": "10.12", + "full": "Mac OS X 10.12" + }, + "device": { + "name": "Mac" + }, + "version": "50.0." + } + }, + { + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, + "url": { + "path": "/hello", + "original": "/hello" + }, + "tags": [ + "preserve_original_event" + ], + "apache": { + "access": {} + }, + "@timestamp": "2016-12-26T16:22:13.000Z", + "ecs": { + "version": "1.12.0" + }, + "http": { + "request": { + "method": "GET", + "referrer": "-" + }, + "version": "1.1", + "response": { + "body": { + "bytes": 499 + }, + "status_code": 404 + } + }, + "event": { + "ingested": "2021-12-09T13:30:31.835563600Z", + "original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", + "category": "web", + "kind": "event", + "created": "2020-04-28T11:07:58.223Z", + "outcome": "failure" + }, + "user": { + "name": "-" + }, + "user_agent": { + "name": "Firefox", + "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0", + "os": { + "name": "Mac OS X", + "version": "10.12", + "full": "Mac OS X 10.12" + }, + "device": { + "name": "Mac" + }, + "version": "50.0." + } + }, + { + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, + "url": { + "path": "/crap", + "original": "/crap" + }, + "tags": [ + "preserve_original_event" + ], + "apache": { + "access": {} + }, + "@timestamp": "2016-12-26T16:22:17.000Z", + "ecs": { + "version": "1.12.0" + }, + "http": { + "request": { + "method": "GET", + "referrer": "-" + }, + "version": "1.1", + "response": { + "body": { + "bytes": 499 + }, + "status_code": 404 + } + }, + "event": { + "ingested": "2021-12-09T13:30:31.835568100Z", + "original": "192.168.33.1 - - [26/Dec/2016:16:22:17 +0000] \"GET /crap HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", + "category": "web", + "kind": "event", + "created": "2020-04-28T11:07:58.223Z", + "outcome": "failure" + }, + "user": { + "name": "-" + }, + "user_agent": { + "name": "Firefox", + "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0", + "os": { + "name": "Mac OS X", + "version": "10.12", + "full": "Mac OS X 10.12" + }, + "device": { + "name": "Mac" + }, + "version": "50.0." + } + } + ] +} \ No newline at end of file diff --git a/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log b/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log new file mode 100644 index 0000000000..64a432e4a8 --- /dev/null +++ b/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log @@ -0,0 +1 @@ +vhost1.domaine.fr 192.168.33.2 - - [26/Dec/2016:16:22:14 +0000] "GET /hello HTTP/1.1" 404 499 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0" diff --git a/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json b/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json new file mode 100644 index 0000000000..8b6a8cbbef --- /dev/null +++ b/test/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json @@ -0,0 +1,64 @@ +{ + "expected": [ + { + "destination": { + "domain": "vhost1.domaine.fr" + }, + "source": { + "ip": "192.168.33.2" + }, + "url": { + "path": "/hello", + "original": "/hello", + "domain": "vhost1.domaine.fr" + }, + "tags": [ + "preserve_original_event" + ], + "apache": { + "access": {} + }, + "@timestamp": "2016-12-26T16:22:14.000Z", + "ecs": { + "version": "1.12.0" + }, + "http": { + "request": { + "method": "GET", + "referrer": "-" + }, + "version": "1.1", + "response": { + "body": { + "bytes": 499 + }, + "status_code": 404 + } + }, + "event": { + "ingested": "2021-12-09T13:30:33.387841500Z", + "original": "vhost1.domaine.fr 192.168.33.2 - - [26/Dec/2016:16:22:14 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", + "category": "web", + "kind": "event", + "created": "2020-04-28T11:07:58.223Z", + "outcome": "failure" + }, + "user": { + "name": "-" + }, + "user_agent": { + "name": "Firefox", + "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0", + "os": { + "name": "Mac OS X", + "version": "10.12", + "full": "Mac OS X 10.12" + }, + "device": { + "name": "Mac" + }, + "version": "50.0." + } + } + ] +} \ No newline at end of file diff --git a/test/packages/apache/data_stream/access/_dev/test/pipeline/test-common-config.yml b/test/packages/apache/data_stream/access/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 0000000000..3cabcf9fb8 --- /dev/null +++ b/test/packages/apache/data_stream/access/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,6 @@ +dynamic_fields: + event.ingested: ".*" +fields: + "@timestamp": "2020-04-28T11:07:58.223Z" + tags: + - preserve_original_event diff --git a/test/packages/apache/data_stream/access/agent/stream/httpjson.yml.hbs b/test/packages/apache/data_stream/access/agent/stream/httpjson.yml.hbs new file mode 100644 index 0000000000..96e6d31467 --- /dev/null +++ b/test/packages/apache/data_stream/access/agent/stream/httpjson.yml.hbs @@ -0,0 +1,64 @@ +config_version: "2" +interval: {{interval}} +{{#unless token}} +{{#if username}} +{{#if password}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +{{/if}} +{{/if}} +{{/unless}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: |- + {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +{{#unless username}} +{{#unless password}} +{{#if token}} + - set: + target: header.Authorization + value: {{token}} +{{/if}} +{{/unless}} +{{/unless}} +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/test/packages/apache/data_stream/access/agent/stream/log.yml.hbs b/test/packages/apache/data_stream/access/agent/stream/log.yml.hbs index 0146a6e9c6..c6e5ed4c73 100644 --- a/test/packages/apache/data_stream/access/agent/stream/log.yml.hbs +++ b/test/packages/apache/data_stream/access/agent/stream/log.yml.hbs @@ -2,9 +2,18 @@ paths: {{#each paths as |path i|}} - {{path}} {{/each}} +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} exclude_files: [".gz$"] +{{#if processors}} processors: - - add_fields: - target: '' - fields: - ecs.version: 1.5.0 \ No newline at end of file +{{processors}} +{{/if}} \ No newline at end of file diff --git a/test/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml b/test/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml index 9e0d5272be..dadfb3a493 100644 --- a/test/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml +++ b/test/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml @@ -1,101 +1,152 @@ --- description: "Pipeline for parsing Apache HTTP Server access logs. Requires the geoip and user_agent plugins." - processors: -- grok: - field: message - patterns: - - '%{IPORHOST:destination.domain} %{IPORHOST:source.ip} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\] - "(?:%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}|-)?" - %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( - "%{DATA:http.request.referrer}")?( "%{DATA:user_agent.original}")?' - - '%{IPORHOST:source.address} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\] - "(?:%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}|-)?" - %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( - "%{DATA:http.request.referrer}")?( "%{DATA:user_agent.original}")?' - - '%{IPORHOST:source.address} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\] - "-" %{NUMBER:http.response.status_code:long} -' - - \[%{HTTPDATE:apache.access.time}\] %{IPORHOST:source.address} %{DATA:apache.access.ssl.protocol} - %{DATA:apache.access.ssl.cipher} "%{WORD:http.request.method} %{DATA:url.original} - HTTP/%{NUMBER:http.version}" (-|%{NUMBER:http.response.body.bytes:long}) - ignore_missing: true -- remove: - field: message -- set: - field: event.kind - value: event -- set: - field: event.category - value: web -- set: - field: event.outcome - value: success - if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400" -- set: - field: event.outcome - value: failure - if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code > 399" -- grok: - field: source.address - ignore_missing: true - patterns: - - ^(%{IP:source.ip}|%{HOSTNAME:source.domain})$ -- rename: - field: '@timestamp' - target_field: event.created -- date: - field: apache.access.time - target_field: '@timestamp' - formats: - - dd/MMM/yyyy:H:m:s Z - ignore_failure: true -- remove: - field: apache.access.time - ignore_failure: true -- user_agent: - field: user_agent.original - ignore_failure: true -- geoip: - field: source.ip - target_field: source.geo - ignore_missing: true -- geoip: - database_file: GeoLite2-ASN.mmdb - field: source.ip - target_field: source.as - properties: - - asn - - organization_name - ignore_missing: true -- rename: - field: source.as.asn - target_field: source.as.number - ignore_missing: true -- rename: - field: source.as.organization_name - target_field: source.as.organization.name - ignore_missing: true -- set: - field: tls.cipher - value: '{{apache.access.ssl.cipher}}' - if: ctx?.apache?.access?.ssl?.cipher != null - -- script: - lang: painless - if: ctx?.apache?.access?.ssl?.protocol != null - source: >- - def parts = ctx.apache.access.ssl.protocol.toLowerCase().splitOnToken("v"); - if (parts.length != 2) { - return; - } - if (parts[1].contains(".")) { - ctx.tls.version = parts[1]; - } else { - ctx.tls.version = parts[1] + ".0"; - } - ctx.tls.version_protocol = parts[0]; - + - pipeline: + if: ctx.message.startsWith('{') + name: '{{ IngestPipeline "third-party" }}' + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + - set: + field: ecs.version + value: '1.12.0' + - rename: + field: message + target_field: event.original + - grok: + field: event.original + patterns: + - '%{IPORHOST:destination.domain} %{IPORHOST:source.ip} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\] + "(?:%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}|-)?" + %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( + "%{DATA:http.request.referrer}")?( "%{DATA:user_agent.original}")?' + - '%{IPORHOST:source.address} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\] + "(?:%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}|-)?" + %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( + "%{DATA:http.request.referrer}")?( "%{DATA:user_agent.original}")?' + - '%{IPORHOST:source.address} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\] + "-" %{NUMBER:http.response.status_code:long} -' + - \[%{HTTPDATE:apache.access.time}\] %{IPORHOST:source.address} %{DATA:apache.access.ssl.protocol} + %{DATA:apache.access.ssl.cipher} "%{WORD:http.request.method} %{DATA:_tmp.url_orig} + HTTP/%{NUMBER:http.version}" (-|%{NUMBER:http.response.body.bytes:long}) + ignore_missing: true + - uri_parts: + field: _tmp.url_orig + ignore_failure: true + - remove: + field: + - _tmp + ignore_missing: true + - set: + field: url.domain + value: "{{destination.domain}}" + if: ctx.url?.domain == null && ctx.destination?.domain != null + - set: + field: event.kind + value: event + - set: + field: event.category + value: web + - set: + field: event.outcome + value: success + if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400" + - set: + field: event.outcome + value: failure + if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code > 399" + - grok: + field: source.address + ignore_missing: true + patterns: + - ^(%{IP:source.ip}|%{HOSTNAME:source.domain})$ + - remove: + field: event.created + ignore_missing: true + ignore_failure: true + - rename: + field: '@timestamp' + target_field: event.created + - date: + field: apache.access.time + target_field: '@timestamp' + formats: + - dd/MMM/yyyy:H:m:s Z + ignore_failure: true + - remove: + field: apache.access.time + ignore_failure: true + - user_agent: + field: user_agent.original + ignore_failure: true + - geoip: + field: source.ip + target_field: source.geo + ignore_missing: true + - geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true + - rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true + - rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true + - set: + field: tls.cipher + value: '{{apache.access.ssl.cipher}}' + if: ctx?.apache?.access?.ssl?.cipher != null + - script: + lang: painless + if: ctx?.apache?.access?.ssl?.protocol != null + source: >- + def parts = ctx.apache.access.ssl.protocol.toLowerCase().splitOnToken("v"); + if (parts.length != 2) { + return; + } + if (parts[1].contains(".")) { + ctx.tls.version = parts[1]; + } else { + ctx.tls.version = parts[1] + ".0"; + } + ctx.tls.version_protocol = parts[0]; + - script: + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx); + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: -- set: - field: error.message - value: '{{ _ingest.on_failure_message }}' + - set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/test/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/third-party.yml b/test/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/third-party.yml new file mode 100644 index 0000000000..42a2ca83ff --- /dev/null +++ b/test/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/third-party.yml @@ -0,0 +1,42 @@ +--- +description: Pipeline for parsing Apache HTTP Server logs from third party api +processors: + - json: + field: message + target_field: json + - drop: + if: ctx.json?.result == null + - fingerprint: + fields: + - json.result._cd + - json.result._indextime + - json.result._raw + - json.result._time + - json.result.host + - json.result.source + target_field: '_id' + ignore_missing: true + - set: + copy_from: json.result._raw + field: message + ignore_empty_value: true + - set: + copy_from: json.result.host + field: host.name + ignore_empty_value: true + - set: + copy_from: json.result.source + field: file.path + ignore_empty_value: true + - remove: + field: + - json + ignore_missing: true +on_failure: + - append: + field: error.message + value: >- + error in third-party pipeline: + error in [{{_ingest.on_failure_processor_type}}] processor{{#_ingest.on_failure_processor_tag}} + with tag [{{_ingest.on_failure_processor_tag }}]{{/_ingest.on_failure_processor_tag}} + {{ _ingest.on_failure_message }} diff --git a/test/packages/apache/data_stream/access/fields/agent.yml b/test/packages/apache/data_stream/access/fields/agent.yml new file mode 100644 index 0000000000..e313ec8287 --- /dev/null +++ b/test/packages/apache/data_stream/access/fields/agent.yml @@ -0,0 +1,204 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + +- name: input.type + type: keyword + description: Input type +- name: log.offset + type: long + description: Log offset diff --git a/test/packages/apache/data_stream/access/fields/base-fields.yml b/test/packages/apache/data_stream/access/fields/base-fields.yml index 7c798f4534..7a1b27e3da 100644 --- a/test/packages/apache/data_stream/access/fields/base-fields.yml +++ b/test/packages/apache/data_stream/access/fields/base-fields.yml @@ -10,3 +10,11 @@ - name: '@timestamp' type: date description: Event timestamp. +- name: event.module + type: constant_keyword + description: Event module + value: apache +- name: event.dataset + type: constant_keyword + description: Event dataset + value: apache.access diff --git a/test/packages/apache/data_stream/access/fields/ecs.yml b/test/packages/apache/data_stream/access/fields/ecs.yml index 53f81b3c85..12993b0268 100644 --- a/test/packages/apache/data_stream/access/fields/ecs.yml +++ b/test/packages/apache/data_stream/access/fields/ecs.yml @@ -1,199 +1,104 @@ -- name: message +- external: ecs + name: destination.domain +- external: ecs + name: ecs.version +- external: ecs + name: error.message +- external: ecs + name: event.category +- external: ecs + name: event.created +- external: ecs + name: event.kind +- external: ecs + name: event.outcome +- external: ecs + name: file.path +- external: ecs + name: http.request.method +- external: ecs + name: http.request.referrer +- external: ecs + name: http.response.body.bytes +- external: ecs + name: http.response.status_code +- external: ecs + name: http.version +- external: ecs + name: log.file.path +- external: ecs + name: log.level +- external: ecs + name: message +- external: ecs + name: process.pid +- external: ecs + name: process.thread.id +- external: ecs + name: source.address +- external: ecs + name: source.as.number +- external: ecs + name: source.as.organization.name +- external: ecs + name: source.domain +- external: ecs + name: source.geo.city_name +- external: ecs + name: source.geo.continent_name +- external: ecs + name: source.geo.country_iso_code +- external: ecs + name: source.geo.country_name +- description: Longitude and latitude. level: core - type: text - description: |- - For log events the message field contains the log message, optimized for viewing in a log viewer. - For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. - If multiple messages exist, they can be combined into one message. -- name: http - title: HTTP - group: 2 - type: group - fields: - - name: request.method - level: extended - type: keyword - description: |- - HTTP request method. - Prior to ECS 1.6.0 the following guidance was provided: - "The field value must be normalized to lowercase for querying." - As of ECS 1.6.0, the guidance is deprecated because the original case of the method may be useful in anomaly detection. Original case will be mandated in ECS 2.0.0 - ignore_above: 1024 - - name: request.referrer - level: extended - type: keyword - description: Referrer for this HTTP request. - ignore_above: 1024 - - name: response.body.bytes - level: extended - type: long - format: bytes - description: Size in bytes of the response body. - unit: byte - metric_type: gauge - - name: response.status_code - level: extended - type: long - format: string - description: HTTP response status code. - - name: version - level: extended - type: keyword - description: HTTP version. - ignore_above: 1024 -- name: log - title: Log - group: 2 - type: group - fields: - - name: level - level: core - type: keyword - description: |- - Original log level of the log event. - If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity). - Some examples are `warn`, `err`, `i`, `informational`. - ignore_above: 1024 -- name: process - title: Process - group: 2 - type: group - fields: - - name: pid - level: core - type: long - format: string - description: Process id. - - name: thread.id - level: extended - type: long - format: string - description: Thread ID. -- name: source - title: Source - group: 2 - type: group - fields: - - name: address - level: extended - type: keyword - description: |- - Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. - Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. - ignore_above: 1024 - - name: geo.city_name - level: core - type: keyword - description: City name. - ignore_above: 1024 - - name: geo.continent_name - level: core - type: keyword - description: Name of the continent. - ignore_above: 1024 - - name: geo.country_iso_code - level: core - type: keyword - description: Country ISO code. - ignore_above: 1024 - - name: geo.location - level: core - type: geo_point - description: Longitude and latitude. - - name: geo.region_iso_code - level: core - type: keyword - description: Region ISO code. - ignore_above: 1024 - - name: geo.region_name - level: core - type: keyword - description: Region name. - ignore_above: 1024 -- name: url - title: URL - group: 2 - type: group - fields: - - name: original - level: extended - type: keyword - description: |- - Unmodified original url as seen in the event source. - Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. - This field is meant to represent the URL as it was observed, complete or not. - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false -- name: user - title: User - group: 2 - type: group - fields: - - name: name - level: core - type: keyword - description: Short name or login of the user. - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false -- name: user_agent - title: User agent - group: 2 - type: group - fields: - - name: device.name - level: extended - type: keyword - description: Name of the device. - ignore_above: 1024 - - name: name - level: extended - type: keyword - description: Name of the user agent. - ignore_above: 1024 - - name: original - level: extended - type: keyword - description: Unparsed user_agent string. - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - - name: os.name - level: extended - type: keyword - description: Operating system name, without the version. - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - - name: os.version - level: extended - type: keyword - ignore_above: 1024 - description: Operating system version as a raw string. - - name: version - level: extended - type: keyword - ignore_above: 1024 - description: Version of the user agent. -- name: ecs.version - type: keyword -- name: source.ip - type: ip -- name: log.file.path - type: keyword -- name: log.offset - type: long -- name: input.type - type: keyword + name: source.geo.location + type: geo_point +- external: ecs + name: source.geo.region_iso_code +- external: ecs + name: source.geo.region_name +- external: ecs + name: source.ip +- external: ecs + name: tags +- external: ecs + name: tls.cipher +- external: ecs + name: tls.version +- external: ecs + name: tls.version_protocol +- external: ecs + name: url.domain +- external: ecs + name: url.extension +- external: ecs + name: url.original +- external: ecs + name: url.path +- external: ecs + name: url.query +- external: ecs + name: user.name +- external: ecs + name: user_agent.device.name +- external: ecs + name: user_agent.device.name +- external: ecs + name: user_agent.name +- external: ecs + name: user_agent.name +- external: ecs + name: user_agent.original +- external: ecs + name: user_agent.original +- external: ecs + name: user_agent.os.full +- external: ecs + name: user_agent.os.name +- external: ecs + name: user_agent.os.name +- external: ecs + name: user_agent.os.version +- external: ecs + name: user_agent.version diff --git a/test/packages/apache/data_stream/access/manifest.yml b/test/packages/apache/data_stream/access/manifest.yml index a339bc28d7..dc93238a33 100644 --- a/test/packages/apache/data_stream/access/manifest.yml +++ b/test/packages/apache/data_stream/access/manifest.yml @@ -1,5 +1,4 @@ title: Apache access logs -release: experimental type: logs streams: - input: logfile @@ -14,6 +13,75 @@ streams: - /var/log/apache2/access.log* - /var/log/apache2/other_vhosts_access.log* - /var/log/httpd/access_log* + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - apache-access + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + template_path: log.yml.hbs title: Apache access logs description: Collect Apache access logs + - input: httpjson + title: Apache access logs via Splunk Enterprise REST API + description: Collect apache access logs via Splunk Enterprise REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query Splunk Enterprise REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Splunk search string + show_user: true + required: true + default: "search sourcetype=\"access*\"" + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded + - apache-access + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + diff --git a/test/packages/apache/data_stream/access/sample_event.json b/test/packages/apache/data_stream/access/sample_event.json index 04f2842882..faf5bb50af 100644 --- a/test/packages/apache/data_stream/access/sample_event.json +++ b/test/packages/apache/data_stream/access/sample_event.json @@ -1,85 +1,84 @@ { - "@timestamp": "2021-03-11T00:50:31.000Z", "agent": { - "ephemeral_id": "c024babb-4b5e-4848-8ab7-808871a3b0e7", - "hostname": "docker-fleet-agent", - "id": "998f41ec-bd9a-4eb8-b03e-6b2106ad8a82", - "name": "docker-fleet-agent", + "hostname": "4942ef7a8cfc", + "name": "4942ef7a8cfc", + "id": "73de002e-d848-49c7-829d-e903959d0d44", "type": "filebeat", - "version": "7.13.0" + "ephemeral_id": "e8970288-5c73-40e7-8626-8d297104f4eb", + "version": "7.11.0" + }, + "log": { + "file": { + "path": "/tmp/service_logs/access.log" + }, + "offset": 0 + }, + "elastic_agent": { + "id": "6c69e2bc-7bb3-4bac-b7e9-41f22558321c", + "version": "7.11.0", + "snapshot": true + }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "url": { + "original": "/" + }, + "input": { + "type": "log" }, "apache": { "access": {} }, - "data_stream": { - "dataset": "apache.access", - "namespace": "ep", - "type": "logs" - }, + "@timestamp": "2020-12-03T16:25:36.000Z", "ecs": { "version": "1.5.0" }, - "elastic_agent": { - "id": "b34164e3-15e2-4184-9730-2dbe7c8e51d2", - "snapshot": true, - "version": "7.13.0" - }, - "event": { - "category": "web", - "created": "2021-03-11T00:50:44.862Z", - "dataset": "apache.access", - "kind": "event", - "outcome": "success" + "data_stream": { + "namespace": "ep", + "type": "logs", + "dataset": "apache.access" }, "host": { - "architecture": "x86_64", + "hostname": "4942ef7a8cfc", + "os": { + "kernel": "4.9.184-linuxkit", + "codename": "Core", + "name": "CentOS Linux", + "family": "redhat", + "version": "7 (Core)", + "platform": "centos" + }, "containerized": true, - "hostname": "docker-fleet-agent", - "id": "c343f2fc9433c580ea642a287d22d011", "ip": [ - "172.20.0.6" + "192.168.0.4" ], + "name": "4942ef7a8cfc", + "id": "06c26569966fd125c15acac5d7feffb6", "mac": [ - "02:42:ac:14:00:06" + "02:42:c0:a8:00:04" ], - "name": "docker-fleet-agent", - "os": { - "codename": "Core", - "family": "redhat", - "kernel": "4.19.121-linuxkit", - "name": "CentOS Linux", - "platform": "centos", - "type": "linux", - "version": "7 (Core)" - } + "architecture": "x86_64" }, "http": { "request": { "method": "GET" }, "response": { + "status_code": 200, "body": { "bytes": 45 - }, - "status_code": 200 + } }, "version": "1.1" }, - "input": { - "type": "log" - }, - "log": { - "file": { - "path": "/tmp/service_logs/access.log" - }, - "offset": 0 - }, - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, - "url": { - "original": "/" + "event": { + "kind": "event", + "created": "2020-12-03T16:25:53.907Z", + "category": "web", + "dataset": "apache.access", + "outcome": "success" }, "user": { "name": "-" diff --git a/test/packages/apache/data_stream/error/_dev/test/pipeline/test-common-config.yml b/test/packages/apache/data_stream/error/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 0000000000..91c9eaa210 --- /dev/null +++ b/test/packages/apache/data_stream/error/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,6 @@ +dynamic_fields: + event.ingested: ".*" +fields: + event.timezone: "GMT+2" + tags: + - preserve_original_event diff --git a/test/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log b/test/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log new file mode 100644 index 0000000000..b8120aacfd --- /dev/null +++ b/test/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log @@ -0,0 +1,4 @@ +[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico +[Mon Dec 26 16:15:55.103786 2016] [core:notice] [pid 11379] AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd' +[Fri Sep 09 10:42:29.902022 2011] [core:error] [pid 35708:tid 4328636416] [client 89.160.20.156] File does not exist: /usr/local/apache2/htdocs/favicon.ico +[Thu Jun 27 06:58:09.169510 2019] [include:warn] [pid 15934] [client 89.160.20.156:12345] AH01374: mod_include: Options +Includes (or IncludesNoExec) wasn't set, INCLUDES filter removed: /test.html diff --git a/test/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json b/test/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json new file mode 100644 index 0000000000..21df2d32d2 --- /dev/null +++ b/test/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json @@ -0,0 +1,173 @@ +{ + "expected": [ + { + "apache": { + "error": {} + }, + "file": { + "path": "/var/www/favicon.ico" + }, + "@timestamp": "2016-12-26T16:22:08.000+02:00", + "ecs": { + "version": "1.12.0" + }, + "log": { + "level": "error" + }, + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, + "event": { + "ingested": "2021-12-09T13:30:33.868254100Z", + "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", + "category": "web", + "type": "error", + "timezone": "GMT+2", + "kind": "event" + }, + "message": "File does not exist: /var/www/favicon.ico", + "tags": [ + "preserve_original_event" + ] + }, + { + "process": { + "pid": 11379 + }, + "apache": { + "error": { + "module": "core" + } + }, + "@timestamp": "2016-12-26T16:15:55.103+02:00", + "ecs": { + "version": "1.12.0" + }, + "log": { + "level": "notice" + }, + "event": { + "ingested": "2021-12-09T13:30:33.868263600Z", + "original": "[Mon Dec 26 16:15:55.103786 2016] [core:notice] [pid 11379] AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", + "category": "web", + "type": "info", + "timezone": "GMT+2", + "kind": "event" + }, + "message": "AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", + "tags": [ + "preserve_original_event" + ] + }, + { + "process": { + "pid": 35708, + "thread": { + "id": 4328636416 + } + }, + "log": { + "level": "error" + }, + "source": { + "geo": { + "continent_name": "Europe", + "region_iso_code": "SE-AB", + "city_name": "Tumba", + "country_iso_code": "SE", + "country_name": "Sweden", + "region_name": "Stockholm", + "location": { + "lon": 17.8167, + "lat": 59.2 + } + }, + "as": { + "number": 29518, + "organization": { + "name": "Bredband2 AB" + } + }, + "address": "89.160.20.156", + "ip": "89.160.20.156" + }, + "message": "File does not exist: /usr/local/apache2/htdocs/favicon.ico", + "tags": [ + "preserve_original_event" + ], + "apache": { + "error": { + "module": "core" + } + }, + "file": { + "path": "/usr/local/apache2/htdocs/favicon.ico" + }, + "@timestamp": "2011-09-09T10:42:29.902+02:00", + "ecs": { + "version": "1.12.0" + }, + "event": { + "ingested": "2021-12-09T13:30:33.868270Z", + "original": "[Fri Sep 09 10:42:29.902022 2011] [core:error] [pid 35708:tid 4328636416] [client 89.160.20.156] File does not exist: /usr/local/apache2/htdocs/favicon.ico", + "category": "web", + "type": "error", + "timezone": "GMT+2", + "kind": "event" + } + }, + { + "process": { + "pid": 15934 + }, + "apache": { + "error": { + "module": "include" + } + }, + "@timestamp": "2019-06-27T06:58:09.169+02:00", + "ecs": { + "version": "1.12.0" + }, + "log": { + "level": "warn" + }, + "source": { + "geo": { + "continent_name": "Europe", + "region_iso_code": "SE-AB", + "city_name": "Tumba", + "country_iso_code": "SE", + "country_name": "Sweden", + "region_name": "Stockholm", + "location": { + "lon": 17.8167, + "lat": 59.2 + } + }, + "as": { + "number": 29518, + "organization": { + "name": "Bredband2 AB" + } + }, + "address": "89.160.20.156", + "port": 12345, + "ip": "89.160.20.156" + }, + "event": { + "ingested": "2021-12-09T13:30:33.868275800Z", + "original": "[Thu Jun 27 06:58:09.169510 2019] [include:warn] [pid 15934] [client 89.160.20.156:12345] AH01374: mod_include: Options +Includes (or IncludesNoExec) wasn't set, INCLUDES filter removed: /test.html", + "category": "web", + "type": "error", + "timezone": "GMT+2", + "kind": "event" + }, + "message": "AH01374: mod_include: Options +Includes (or IncludesNoExec) wasn't set, INCLUDES filter removed: /test.html", + "tags": [ + "preserve_original_event" + ] + } + ] +} \ No newline at end of file diff --git a/test/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log b/test/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log new file mode 100644 index 0000000000..428ba85a7f --- /dev/null +++ b/test/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log @@ -0,0 +1,2 @@ +[Mon Dec 26 16:15:55.103522 2016] [mpm_prefork:notice] [pid 11379] AH00163: Apache/2.4.23 (Unix) configured -- resuming normal operations +[Mon Dec 26 16:15:55.103786 2016] [core:notice] [pid 11379] AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd' diff --git a/test/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json b/test/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json new file mode 100644 index 0000000000..15400a6c97 --- /dev/null +++ b/test/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json @@ -0,0 +1,62 @@ +{ + "expected": [ + { + "process": { + "pid": 11379 + }, + "apache": { + "error": { + "module": "mpm_prefork" + } + }, + "@timestamp": "2016-12-26T16:15:55.103+02:00", + "ecs": { + "version": "1.12.0" + }, + "log": { + "level": "notice" + }, + "event": { + "ingested": "2021-12-09T13:30:34.149405700Z", + "original": "[Mon Dec 26 16:15:55.103522 2016] [mpm_prefork:notice] [pid 11379] AH00163: Apache/2.4.23 (Unix) configured -- resuming normal operations", + "category": "web", + "type": "info", + "timezone": "GMT+2", + "kind": "event" + }, + "message": "AH00163: Apache/2.4.23 (Unix) configured -- resuming normal operations", + "tags": [ + "preserve_original_event" + ] + }, + { + "process": { + "pid": 11379 + }, + "apache": { + "error": { + "module": "core" + } + }, + "@timestamp": "2016-12-26T16:15:55.103+02:00", + "ecs": { + "version": "1.12.0" + }, + "log": { + "level": "notice" + }, + "event": { + "ingested": "2021-12-09T13:30:34.149429600Z", + "original": "[Mon Dec 26 16:15:55.103786 2016] [core:notice] [pid 11379] AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", + "category": "web", + "type": "info", + "timezone": "GMT+2", + "kind": "event" + }, + "message": "AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", + "tags": [ + "preserve_original_event" + ] + } + ] +} \ No newline at end of file diff --git a/test/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log b/test/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log new file mode 100644 index 0000000000..64a89473fb --- /dev/null +++ b/test/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log @@ -0,0 +1 @@ +[Wed Oct 20 19:20:59.121211 2021] [rewrite:trace3] [pid 121591:tid 140413273032448] mod_rewrite.c(470): [client 10.121.192.8:38350] 10.121.192.8 - - [dev.elastic.co/sid#55a374e851c8][rid#7fb438083ac0/initial] applying pattern '^/import/?(.*)$' to uri '/' diff --git a/test/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json b/test/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json new file mode 100644 index 0000000000..7b5f6e1b4b --- /dev/null +++ b/test/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json @@ -0,0 +1,36 @@ +{ + "expected": [ + { + "process": { + "pid": 121591, + "thread": { + "id": 140413273032448 + } + }, + "apache": { + "error": { + "module": "rewrite" + } + }, + "@timestamp": "2021-10-20T19:20:59.121+02:00", + "ecs": { + "version": "1.12.0" + }, + "log": { + "level": "trace3" + }, + "event": { + "ingested": "2021-12-09T13:30:34.228018700Z", + "original": "[Wed Oct 20 19:20:59.121211 2021] [rewrite:trace3] [pid 121591:tid 140413273032448] mod_rewrite.c(470): [client 10.121.192.8:38350] 10.121.192.8 - - [dev.elastic.co/sid#55a374e851c8][rid#7fb438083ac0/initial] applying pattern '^/import/?(.*)$' to uri '/'", + "category": "web", + "type": "info", + "timezone": "GMT+2", + "kind": "event" + }, + "message": "mod_rewrite.c(470): [client 10.121.192.8:38350] 10.121.192.8 - - [dev.elastic.co/sid#55a374e851c8][rid#7fb438083ac0/initial] applying pattern '^/import/?(.*)$' to uri '/'", + "tags": [ + "preserve_original_event" + ] + } + ] +} \ No newline at end of file diff --git a/test/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log b/test/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log new file mode 100644 index 0000000000..e457f0bc78 --- /dev/null +++ b/test/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log @@ -0,0 +1,7 @@ +[Mon Dec 26 16:17:53 2016] [notice] Apache/2.2.22 (Ubuntu) configured -- resuming normal operations +[Mon Dec 26 16:22:00 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico, referer: http://192.168.33.72/ +[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico +[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico +[Mon Dec 26 16:22:10 2016] [error] [client 192.168.33.1] File does not exist: /var/www/test +[Mon Dec 26 16:22:13 2016] [error] [client 192.168.33.1] File does not exist: /var/www/hello +[Mon Dec 26 16:22:17 2016] [error] [client 192.168.33.1] File does not exist: /var/www/crap diff --git a/test/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json b/test/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json new file mode 100644 index 0000000000..92c843f590 --- /dev/null +++ b/test/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json @@ -0,0 +1,219 @@ +{ + "expected": [ + { + "apache": { + "error": {} + }, + "@timestamp": "2016-12-26T16:17:53.000+02:00", + "ecs": { + "version": "1.12.0" + }, + "log": { + "level": "notice" + }, + "event": { + "ingested": "2021-12-09T13:30:34.283841100Z", + "original": "[Mon Dec 26 16:17:53 2016] [notice] Apache/2.2.22 (Ubuntu) configured -- resuming normal operations", + "category": "web", + "type": "info", + "timezone": "GMT+2", + "kind": "event" + }, + "message": "Apache/2.2.22 (Ubuntu) configured -- resuming normal operations", + "tags": [ + "preserve_original_event" + ] + }, + { + "log": { + "level": "error" + }, + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, + "message": "File does not exist: /var/www/favicon.ico, referer: http://192.168.33.72/", + "tags": [ + "preserve_original_event" + ], + "apache": { + "error": {} + }, + "file": { + "path": "/var/www/favicon.ico" + }, + "@timestamp": "2016-12-26T16:22:00.000+02:00", + "ecs": { + "version": "1.12.0" + }, + "http": { + "request": { + "referrer": "http://192.168.33.72/" + } + }, + "event": { + "ingested": "2021-12-09T13:30:34.283849400Z", + "original": "[Mon Dec 26 16:22:00 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico, referer: http://192.168.33.72/", + "category": "web", + "type": "error", + "timezone": "GMT+2", + "kind": "event" + } + }, + { + "apache": { + "error": {} + }, + "file": { + "path": "/var/www/favicon.ico" + }, + "@timestamp": "2016-12-26T16:22:08.000+02:00", + "ecs": { + "version": "1.12.0" + }, + "log": { + "level": "error" + }, + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, + "event": { + "ingested": "2021-12-09T13:30:34.283853Z", + "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", + "category": "web", + "type": "error", + "timezone": "GMT+2", + "kind": "event" + }, + "message": "File does not exist: /var/www/favicon.ico", + "tags": [ + "preserve_original_event" + ] + }, + { + "apache": { + "error": {} + }, + "file": { + "path": "/var/www/favicon.ico" + }, + "@timestamp": "2016-12-26T16:22:08.000+02:00", + "ecs": { + "version": "1.12.0" + }, + "log": { + "level": "error" + }, + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, + "event": { + "ingested": "2021-12-09T13:30:34.283857200Z", + "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", + "category": "web", + "type": "error", + "timezone": "GMT+2", + "kind": "event" + }, + "message": "File does not exist: /var/www/favicon.ico", + "tags": [ + "preserve_original_event" + ] + }, + { + "apache": { + "error": {} + }, + "file": { + "path": "/var/www/test" + }, + "@timestamp": "2016-12-26T16:22:10.000+02:00", + "ecs": { + "version": "1.12.0" + }, + "log": { + "level": "error" + }, + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, + "event": { + "ingested": "2021-12-09T13:30:34.283862600Z", + "original": "[Mon Dec 26 16:22:10 2016] [error] [client 192.168.33.1] File does not exist: /var/www/test", + "category": "web", + "type": "error", + "timezone": "GMT+2", + "kind": "event" + }, + "message": "File does not exist: /var/www/test", + "tags": [ + "preserve_original_event" + ] + }, + { + "apache": { + "error": {} + }, + "file": { + "path": "/var/www/hello" + }, + "@timestamp": "2016-12-26T16:22:13.000+02:00", + "ecs": { + "version": "1.12.0" + }, + "log": { + "level": "error" + }, + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, + "event": { + "ingested": "2021-12-09T13:30:34.283867900Z", + "original": "[Mon Dec 26 16:22:13 2016] [error] [client 192.168.33.1] File does not exist: /var/www/hello", + "category": "web", + "type": "error", + "timezone": "GMT+2", + "kind": "event" + }, + "message": "File does not exist: /var/www/hello", + "tags": [ + "preserve_original_event" + ] + }, + { + "apache": { + "error": {} + }, + "file": { + "path": "/var/www/crap" + }, + "@timestamp": "2016-12-26T16:22:17.000+02:00", + "ecs": { + "version": "1.12.0" + }, + "log": { + "level": "error" + }, + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, + "event": { + "ingested": "2021-12-09T13:30:34.283873300Z", + "original": "[Mon Dec 26 16:22:17 2016] [error] [client 192.168.33.1] File does not exist: /var/www/crap", + "category": "web", + "type": "error", + "timezone": "GMT+2", + "kind": "event" + }, + "message": "File does not exist: /var/www/crap", + "tags": [ + "preserve_original_event" + ] + } + ] +} \ No newline at end of file diff --git a/test/packages/apache/data_stream/error/_dev/test/system/test-default-config.yml b/test/packages/apache/data_stream/error/_dev/test/system/test-default-config.yml index 6304ef30f5..ec7356ee93 100644 --- a/test/packages/apache/data_stream/error/_dev/test/system/test-default-config.yml +++ b/test/packages/apache/data_stream/error/_dev/test/system/test-default-config.yml @@ -1,10 +1,5 @@ -skip: - reason: testing skip - link: https://github.com/elastic/integrations/issues/123456789 vars: ~ data_stream: vars: paths: - # This path should cause the test to fail if the skip feature - # stops working as expected. - - "{{SERVICE_LOGS_DIR}}/non-existent.log*" + - "{{SERVICE_LOGS_DIR}}/error.log*" diff --git a/test/packages/apache/data_stream/error/agent/stream/httpjson.yml.hbs b/test/packages/apache/data_stream/error/agent/stream/httpjson.yml.hbs new file mode 100644 index 0000000000..d96b2f6048 --- /dev/null +++ b/test/packages/apache/data_stream/error/agent/stream/httpjson.yml.hbs @@ -0,0 +1,64 @@ +config_version: 2 +interval: {{interval}} +{{#unless token}} +{{#if username}} +{{#if password}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +{{/if}} +{{/if}} +{{/unless}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: |- + {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +{{#unless username}} +{{#unless password}} +{{#if token}} + - set: + target: header.Authorization + value: {{token}} +{{/if}} +{{/unless}} +{{/unless}} +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} \ No newline at end of file diff --git a/test/packages/apache/data_stream/error/agent/stream/log.yml.hbs b/test/packages/apache/data_stream/error/agent/stream/log.yml.hbs index 9a26f86f59..781c90981a 100644 --- a/test/packages/apache/data_stream/error/agent/stream/log.yml.hbs +++ b/test/packages/apache/data_stream/error/agent/stream/log.yml.hbs @@ -3,9 +3,18 @@ paths: - {{path}} {{/each}} exclude_files: [".gz$"] +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} processors: - - add_locale: ~ - - add_fields: - target: '' - fields: - ecs.version: 1.5.0 \ No newline at end of file +- add_locale: ~ +{{#if processors}} +{{processors}} +{{/if}} \ No newline at end of file diff --git a/test/packages/apache/data_stream/error/elasticsearch/ingest_pipeline/default.yml b/test/packages/apache/data_stream/error/elasticsearch/ingest_pipeline/default.yml index a39c890f69..6c4bba6c79 100644 --- a/test/packages/apache/data_stream/error/elasticsearch/ingest_pipeline/default.yml +++ b/test/packages/apache/data_stream/error/elasticsearch/ingest_pipeline/default.yml @@ -1,86 +1,140 @@ --- description: Pipeline for parsing apache error logs processors: -- grok: - field: message - patterns: - - \[%{APACHE_TIME:apache.error.timestamp}\] \[%{LOGLEVEL:log.level}\]( \[client - %{IPORHOST:source.address}(:%{POSINT:source.port})?\])? %{GREEDYDATA:message} - - \[%{APACHE_TIME:apache.error.timestamp}\] \[%{DATA:apache.error.module}:%{LOGLEVEL:log.level}\] - \[pid %{NUMBER:process.pid:long}(:tid %{NUMBER:process.thread.id:long})?\]( - \[client %{IPORHOST:source.address}(:%{POSINT:source.port})?\])? %{GREEDYDATA:message} - pattern_definitions: - APACHE_TIME: '%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}' - ignore_missing: true -- date: - if: ctx.event.timezone == null - field: apache.error.timestamp - target_field: '@timestamp' - formats: - - EEE MMM dd H:m:s yyyy - - EEE MMM dd H:m:s.SSSSSS yyyy - on_failure: - - append: - field: error.message - value: '{{ _ingest.on_failure_message }}' -- date: - if: ctx.event.timezone != null - field: apache.error.timestamp - target_field: '@timestamp' - formats: - - EEE MMM dd H:m:s yyyy - - EEE MMM dd H:m:s.SSSSSS yyyy - timezone: '{{ event.timezone }}' - on_failure: - - append: - field: error.message - value: '{{ _ingest.on_failure_message }}' -- remove: - field: apache.error.timestamp - ignore_failure: true -- set: - field: event.kind - value: event -- set: - field: event.category - value: web -- script: - if: "ctx?.log?.level != null" - lang: painless - source: >- - def err_levels = ["emerg", "alert", "crit", "error", "warn"]; - if (err_levels.contains(ctx.log.level)) { - ctx.event.type = "error"; - } else { - ctx.event.type = "info"; - } + - pipeline: + if: ctx.message.startsWith('{') + name: '{{ IngestPipeline "third-party" }}' + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + - set: + field: ecs.version + value: '1.12.0' + - rename: + field: message + target_field: event.original + - grok: + field: event.original + patterns: + - \[%{APACHE_TIME:apache.error.timestamp}\] \[%{LOGLEVEL:log.level}\]( \[client + %{IPORHOST:source.address}(:%{POSINT:source.port})?\])? %{GREEDYDATA:message} + - \[%{APACHE_TIME:apache.error.timestamp}\] \[%{DATA:apache.error.module}:%{APACHE_LOGLEVEL:log.level}\] + \[pid %{NUMBER:process.pid:long}(:tid %{NUMBER:process.thread.id:long})?\]( + \[client %{IPORHOST:source.address}(:%{POSINT:source.port})?\])? %{GREEDYDATA:message} + pattern_definitions: + # Apache log level can have numeric sub-levels such as trace1. + APACHE_LOGLEVEL: '%{LOGLEVEL}[0-9]*' + APACHE_TIME: '%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}' + ignore_missing: true + - grok: + field: message + patterns: + - 'File does not exist: %{URIPATH:file.path}, referer: %{URI:http.request.referrer}' + - 'File does not exist: %{URIPATH:file.path}' + ignore_missing: true + ignore_failure: true + - date: + if: ctx.event.timezone == null + field: apache.error.timestamp + target_field: '@timestamp' + formats: + - EEE MMM dd H:m:s yyyy + - EEE MMM dd H:m:s.SSSSSS yyyy + on_failure: + - append: + field: error.message + value: '{{ _ingest.on_failure_message }}' + - date: + if: ctx.event.timezone != null + field: apache.error.timestamp + target_field: '@timestamp' + formats: + - EEE MMM dd H:m:s yyyy + - EEE MMM dd H:m:s.SSSSSS yyyy + timezone: '{{ event.timezone }}' + on_failure: + - append: + field: error.message + value: '{{ _ingest.on_failure_message }}' + - remove: + field: apache.error.timestamp + ignore_failure: true + - set: + field: event.kind + value: event + - set: + field: event.category + value: web + - script: + if: "ctx?.log?.level != null" + lang: painless + source: >- + def err_levels = ["emerg", "alert", "crit", "error", "warn"]; + if (err_levels.contains(ctx.log.level)) { + ctx.event.type = "error"; + } else { + ctx.event.type = "info"; + } -- grok: - field: source.address - ignore_missing: true - patterns: - - ^(%{IP:source.ip}|%{HOSTNAME:source.domain})$ -- geoip: - field: source.ip - target_field: source.geo - ignore_missing: true -- geoip: - database_file: GeoLite2-ASN.mmdb - field: source.ip - target_field: source.as - properties: - - asn - - organization_name - ignore_missing: true -- rename: - field: source.as.asn - target_field: source.as.number - ignore_missing: true -- rename: - field: source.as.organization_name - target_field: source.as.organization.name - ignore_missing: true + - grok: + field: source.address + ignore_missing: true + patterns: + - ^(%{IP:source.ip}|%{HOSTNAME:source.domain})$ + - geoip: + field: source.ip + target_field: source.geo + ignore_missing: true + - geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true + - rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true + - rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true + - convert: + field: source.port + type: long + ignore_missing: true + - script: + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx); + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: -- set: - field: error.message - value: '{{ _ingest.on_failure_message }}' + - set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/test/packages/apache/data_stream/error/elasticsearch/ingest_pipeline/third-party.yml b/test/packages/apache/data_stream/error/elasticsearch/ingest_pipeline/third-party.yml new file mode 100644 index 0000000000..42a2ca83ff --- /dev/null +++ b/test/packages/apache/data_stream/error/elasticsearch/ingest_pipeline/third-party.yml @@ -0,0 +1,42 @@ +--- +description: Pipeline for parsing Apache HTTP Server logs from third party api +processors: + - json: + field: message + target_field: json + - drop: + if: ctx.json?.result == null + - fingerprint: + fields: + - json.result._cd + - json.result._indextime + - json.result._raw + - json.result._time + - json.result.host + - json.result.source + target_field: '_id' + ignore_missing: true + - set: + copy_from: json.result._raw + field: message + ignore_empty_value: true + - set: + copy_from: json.result.host + field: host.name + ignore_empty_value: true + - set: + copy_from: json.result.source + field: file.path + ignore_empty_value: true + - remove: + field: + - json + ignore_missing: true +on_failure: + - append: + field: error.message + value: >- + error in third-party pipeline: + error in [{{_ingest.on_failure_processor_type}}] processor{{#_ingest.on_failure_processor_tag}} + with tag [{{_ingest.on_failure_processor_tag }}]{{/_ingest.on_failure_processor_tag}} + {{ _ingest.on_failure_message }} diff --git a/test/packages/apache/data_stream/error/fields/agent.yml b/test/packages/apache/data_stream/error/fields/agent.yml new file mode 100644 index 0000000000..e313ec8287 --- /dev/null +++ b/test/packages/apache/data_stream/error/fields/agent.yml @@ -0,0 +1,204 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + +- name: input.type + type: keyword + description: Input type +- name: log.offset + type: long + description: Log offset diff --git a/test/packages/apache/data_stream/error/fields/base-fields.yml b/test/packages/apache/data_stream/error/fields/base-fields.yml index 7c798f4534..e134277b8e 100644 --- a/test/packages/apache/data_stream/error/fields/base-fields.yml +++ b/test/packages/apache/data_stream/error/fields/base-fields.yml @@ -10,3 +10,16 @@ - name: '@timestamp' type: date description: Event timestamp. +- name: tags + description: List of keywords used to tag each event. + example: '["production", "env2"]' + ignore_above: 1024 + type: keyword +- name: event.module + type: constant_keyword + description: Event module + value: apache +- name: event.dataset + type: constant_keyword + description: Event dataset + value: apache.error diff --git a/test/packages/apache/data_stream/error/fields/ecs.yml b/test/packages/apache/data_stream/error/fields/ecs.yml index d78c564de5..0a88a11039 100644 --- a/test/packages/apache/data_stream/error/fields/ecs.yml +++ b/test/packages/apache/data_stream/error/fields/ecs.yml @@ -1,185 +1,82 @@ -- name: message +- external: ecs + name: ecs.version +- external: ecs + name: error.message +- external: ecs + name: event.category +- external: ecs + name: event.kind +- external: ecs + name: event.timezone +- external: ecs + name: event.type +- external: ecs + name: file.path +- external: ecs + name: http.request.method +- external: ecs + name: http.request.referrer +- external: ecs + name: http.response.body.bytes +- external: ecs + name: http.response.status_code +- external: ecs + name: http.version +- external: ecs + name: log.file.path +- external: ecs + name: log.level +- external: ecs + name: message +- external: ecs + name: process.pid +- external: ecs + name: process.thread.id +- external: ecs + name: source.address +- external: ecs + name: source.as.number +- external: ecs + name: source.as.organization.name +- external: ecs + name: source.geo.city_name +- external: ecs + name: source.geo.continent_name +- external: ecs + name: source.geo.country_iso_code +- external: ecs + name: source.geo.country_name +- description: Longitude and latitude. level: core - type: text - description: |- - For log events the message field contains the log message, optimized for viewing in a log viewer. - For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. - If multiple messages exist, they can be combined into one message. -- name: http - title: HTTP - group: 2 - type: group - fields: - - name: request.method - level: extended - type: keyword - description: |- - HTTP request method. - Prior to ECS 1.6.0 the following guidance was provided: - "The field value must be normalized to lowercase for querying." - As of ECS 1.6.0, the guidance is deprecated because the original case of the method may be useful in anomaly detection. Original case will be mandated in ECS 2.0.0 - ignore_above: 1024 - - name: request.referrer - level: extended - type: keyword - description: Referrer for this HTTP request. - ignore_above: 1024 - - name: response.body.bytes - level: extended - type: long - format: bytes - description: Size in bytes of the response body. - - name: response.status_code - level: extended - type: long - format: string - description: HTTP response status code. - - name: version - level: extended - type: keyword - description: HTTP version. - ignore_above: 1024 -- name: log - title: Log - group: 2 - type: group - fields: - - name: level - level: core - type: keyword - description: |- - Original log level of the log event. - If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity). - Some examples are `warn`, `err`, `i`, `informational`. - ignore_above: 1024 -- name: process - title: Process - group: 2 - type: group - fields: - - name: pid - level: core - type: long - format: string - description: Process id. - - name: thread.id - level: extended - type: long - format: string - description: Thread ID. -- name: source - title: Source - group: 2 - type: group - fields: - - name: address - level: extended - type: keyword - description: |- - Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. - Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. - ignore_above: 1024 - - name: geo.city_name - level: core - type: keyword - description: City name. - ignore_above: 1024 - - name: geo.continent_name - level: core - type: keyword - description: Name of the continent. - ignore_above: 1024 - - name: geo.country_iso_code - level: core - type: keyword - description: Country ISO code. - ignore_above: 1024 - - name: geo.location - level: core - type: geo_point - description: Longitude and latitude. - - name: geo.region_iso_code - level: core - type: keyword - description: Region ISO code. - ignore_above: 1024 - - name: geo.region_name - level: core - type: keyword - description: Region name. - ignore_above: 1024 -- name: url - title: URL - group: 2 - type: group - fields: - - name: original - level: extended - type: keyword - description: |- - Unmodified original url as seen in the event source. - Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. - This field is meant to represent the URL as it was observed, complete or not. - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false -- name: user - title: User - group: 2 - type: group - fields: - - name: name - level: core - type: keyword - description: Short name or login of the user. - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false -- name: user_agent - title: User agent - group: 2 - type: group - fields: - - name: device.name - level: extended - type: keyword - description: Name of the device. - ignore_above: 1024 - - name: name - level: extended - type: keyword - description: Name of the user agent. - ignore_above: 1024 - - name: original - level: extended - type: keyword - description: Unparsed user_agent string. - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - - name: os.name - level: extended - type: keyword - description: Operating system name, without the version. - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false -- name: ecs.version - type: keyword -- name: log.file.path - type: keyword -- name: log.offset - type: long -- name: input.type - type: keyword + name: source.geo.location + type: geo_point +- external: ecs + name: source.geo.region_iso_code +- external: ecs + name: source.geo.region_name +- external: ecs + name: source.ip +- external: ecs + name: source.port +- external: ecs + name: tags +- external: ecs + name: url.domain +- external: ecs + name: url.extension +- external: ecs + name: url.original +- external: ecs + name: url.path +- external: ecs + name: url.query +- external: ecs + name: user.name +- external: ecs + name: user_agent.device.name +- external: ecs + name: user_agent.name +- external: ecs + name: user_agent.original +- external: ecs + name: user_agent.os.name diff --git a/test/packages/apache/data_stream/error/manifest.yml b/test/packages/apache/data_stream/error/manifest.yml index a50c1e4b60..e0ccdffd68 100644 --- a/test/packages/apache/data_stream/error/manifest.yml +++ b/test/packages/apache/data_stream/error/manifest.yml @@ -1,5 +1,4 @@ title: Apache error logs -release: experimental type: logs streams: - input: logfile @@ -13,6 +12,75 @@ streams: default: - /var/log/apache2/error.log* - /var/log/httpd/error_log* + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - apache-error + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + template_path: log.yml.hbs title: Apache error logs description: Collect Apache error logs + - input: httpjson + title: Apache error logs via Splunk Enterprise REST API + description: Collect apache error logs via Splunk Enterprise REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query Splunk Enterprise REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Splunk search string + show_user: true + required: true + default: search sourcetype=apache:error OR sourcetype=apache_error + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded + - apache-error + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + diff --git a/test/packages/apache/data_stream/error/sample_event.json b/test/packages/apache/data_stream/error/sample_event.json new file mode 100644 index 0000000000..6ddd60ff90 --- /dev/null +++ b/test/packages/apache/data_stream/error/sample_event.json @@ -0,0 +1,74 @@ +{ + "agent": { + "hostname": "4942ef7a8cfc", + "name": "4942ef7a8cfc", + "id": "73de002e-d848-49c7-829d-e903959d0d44", + "ephemeral_id": "e8970288-5c73-40e7-8626-8d297104f4eb", + "type": "filebeat", + "version": "7.11.0" + }, + "process": { + "pid": 1, + "thread": { + "id": 140503592395904 + } + }, + "log": { + "file": { + "path": "/tmp/service_logs/error.log" + }, + "offset": 0, + "level": "notice" + }, + "elastic_agent": { + "id": "6c69e2bc-7bb3-4bac-b7e9-41f22558321c", + "version": "7.11.0", + "snapshot": true + }, + "message": "AH00489: Apache/2.4.46 (Unix) configured -- resuming normal operations", + "input": { + "type": "log" + }, + "@timestamp": "2020-12-03T16:28:16.376Z", + "apache": { + "error": { + "module": "mpm_event" + } + }, + "ecs": { + "version": "1.5.0" + }, + "data_stream": { + "namespace": "ep", + "type": "logs", + "dataset": "apache.error" + }, + "host": { + "hostname": "4942ef7a8cfc", + "os": { + "kernel": "4.9.184-linuxkit", + "codename": "Core", + "name": "CentOS Linux", + "family": "redhat", + "version": "7 (Core)", + "platform": "centos" + }, + "containerized": true, + "ip": [ + "192.168.0.4" + ], + "name": "4942ef7a8cfc", + "id": "06c26569966fd125c15acac5d7feffb6", + "mac": [ + "02:42:c0:a8:00:04" + ], + "architecture": "x86_64" + }, + "event": { + "timezone": "+00:00", + "kind": "event", + "category": "web", + "type": "info", + "dataset": "apache.error" + } +} \ No newline at end of file diff --git a/test/packages/apache/data_stream/status/_dev/test/system/test-default-config.yml b/test/packages/apache/data_stream/status/_dev/test/system/test-default-config.yml index 7493f64a81..83fc84f62e 100644 --- a/test/packages/apache/data_stream/status/_dev/test/system/test-default-config.yml +++ b/test/packages/apache/data_stream/status/_dev/test/system/test-default-config.yml @@ -1,6 +1,3 @@ -skip: - reason: test is failing and needs further investigation but is blocking other PRs. - link: https://github.com/elastic/elastic-package/issues/374 vars: hosts: - http://{{Hostname}}:{{Port}} diff --git a/test/packages/apache/data_stream/status/fields/agent.yml b/test/packages/apache/data_stream/status/fields/agent.yml new file mode 100644 index 0000000000..da4e652c53 --- /dev/null +++ b/test/packages/apache/data_stream/status/fields/agent.yml @@ -0,0 +1,198 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + diff --git a/test/packages/apache/data_stream/status/fields/base-fields.yml b/test/packages/apache/data_stream/status/fields/base-fields.yml index 7c798f4534..2cf8da79f7 100644 --- a/test/packages/apache/data_stream/status/fields/base-fields.yml +++ b/test/packages/apache/data_stream/status/fields/base-fields.yml @@ -10,3 +10,11 @@ - name: '@timestamp' type: date description: Event timestamp. +- name: event.module + type: constant_keyword + description: Event module + value: apache +- name: event.dataset + type: constant_keyword + description: Event dataset + value: apache.status diff --git a/test/packages/apache/data_stream/status/fields/ecs.yml b/test/packages/apache/data_stream/status/fields/ecs.yml index 7e361991e6..28044872f5 100644 --- a/test/packages/apache/data_stream/status/fields/ecs.yml +++ b/test/packages/apache/data_stream/status/fields/ecs.yml @@ -1,6 +1,8 @@ -- name: ecs.version - type: keyword -- name: service.type - type: keyword -- name: service.address - type: keyword +- external: ecs + name: ecs.version +- external: ecs + name: service.address +- external: ecs + name: service.type +- external: ecs + name: error.message diff --git a/test/packages/apache/data_stream/status/fields/fields.yml b/test/packages/apache/data_stream/status/fields/fields.yml index 2c80752a45..5ed7663e2e 100644 --- a/test/packages/apache/data_stream/status/fields/fields.yml +++ b/test/packages/apache/data_stream/status/fields/fields.yml @@ -5,30 +5,38 @@ type: long description: | Total number of access requests. + metric_type: counter - name: total_bytes type: long description: | Total number of bytes served. + metric_type: counter + unit: byte - name: requests_per_sec type: scaled_float description: | Requests per second. + metric_type: gauge - name: bytes_per_sec type: scaled_float description: | Bytes per second. + metric_type: gauge - name: bytes_per_request type: scaled_float description: | Bytes per request. + metric_type: gauge - name: workers.busy type: long description: | Number of busy workers. + metric_type: gauge - name: workers.idle type: long description: | Number of idle workers. + metric_type: gauge - name: uptime type: group fields: @@ -36,10 +44,12 @@ type: long description: | Server uptime in seconds. + metric_type: counter - name: uptime type: long description: | Server uptime. + metric_type: counter - name: cpu type: group fields: @@ -47,22 +57,27 @@ type: scaled_float description: | CPU Load. + metric_type: gauge - name: user type: scaled_float description: | CPU user load. + metric_type: gauge - name: system type: scaled_float description: | System cpu. + metric_type: gauge - name: children_user type: scaled_float description: | CPU of children user. + metric_type: gauge - name: children_system type: scaled_float description: | CPU of children system. + metric_type: gauge - name: connections type: group fields: @@ -70,18 +85,22 @@ type: long description: | Total connections. + metric_type: counter - name: async.writing type: long description: | Async connection writing. + metric_type: gauge - name: async.keep_alive type: long description: | Async keeped alive connections. + metric_type: gauge - name: async.closing type: long description: | Async closed connections. + metric_type: gauge - name: load type: group fields: @@ -89,14 +108,17 @@ type: scaled_float description: | Load average for the last minute. + metric_type: gauge - name: "5" type: scaled_float description: | Load average for the last 5 minutes. + metric_type: gauge - name: "15" type: scaled_float description: | Load average for the last 15 minutes. + metric_type: gauge - name: scoreboard type: group fields: @@ -104,47 +126,59 @@ type: long description: | Starting up. + metric_type: gauge - name: reading_request type: long description: | Reading requests. + metric_type: gauge - name: sending_reply type: long description: | Sending Reply. + metric_type: gauge - name: keepalive type: long description: | Keep alive. + metric_type: gauge - name: dns_lookup type: long description: | Dns Lookups. + metric_type: gauge - name: closing_connection type: long description: | Closing connections. + metric_type: gauge - name: logging type: long description: | Logging + metric_type: gauge - name: gracefully_finishing type: long description: | Gracefully finishing. + metric_type: gauge - name: idle_cleanup type: long description: | Idle cleanups. + metric_type: gauge - name: open_slot type: long description: | Open slots. + metric_type: gauge - name: waiting_for_connection type: long description: | Waiting for connections. + metric_type: gauge - name: total type: long description: | Total. + metric_type: gauge diff --git a/test/packages/apache/data_stream/status/manifest.yml b/test/packages/apache/data_stream/status/manifest.yml index 52de60ae95..f4b63ba327 100644 --- a/test/packages/apache/data_stream/status/manifest.yml +++ b/test/packages/apache/data_stream/status/manifest.yml @@ -1,5 +1,4 @@ title: Apache status metrics -release: experimental type: metrics streams: - input: apache/metrics @@ -10,7 +9,7 @@ streams: multi: false required: true show_user: true - default: 10s + default: 30s - name: server_status_path type: text title: Server Status Path diff --git a/test/packages/apache/data_stream/status/sample_event.json b/test/packages/apache/data_stream/status/sample_event.json index 2088a8eb44..2d23267290 100644 --- a/test/packages/apache/data_stream/status/sample_event.json +++ b/test/packages/apache/data_stream/status/sample_event.json @@ -1,77 +1,101 @@ { - "@timestamp": "2020-06-24T10:19:48.005Z", + "@timestamp": "2020-12-03T16:31:04.445Z", + "data_stream": { + "type": "metrics", + "dataset": "apache.status", + "namespace": "ep" + }, + "elastic_agent": { + "version": "7.11.0", + "id": "6c69e2bc-7bb3-4bac-b7e9-41f22558321c", + "snapshot": true + }, + "host": { + "os": { + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.9.184-linuxkit", + "codename": "Core" + }, + "id": "06c26569966fd125c15acac5d7feffb6", + "name": "4942ef7a8cfc", + "containerized": true, + "ip": [ + "192.168.0.4" + ], + "mac": [ + "02:42:c0:a8:00:04" + ], + "hostname": "4942ef7a8cfc", + "architecture": "x86_64" + }, + "agent": { + "hostname": "4942ef7a8cfc", + "ephemeral_id": "8371d3a3-5321-4436-9fd5-cafcabfe4c57", + "id": "af6f66ef-d7d0-4784-b9bb-3fddbcc151b5", + "name": "4942ef7a8cfc", + "type": "metricbeat", + "version": "7.11.0" + }, "metricset": { "name": "status", - "period": 10000 + "period": 30000 + }, + "service": { + "address": "http://elastic-package-service_apache_1:80/server-status?auto=", + "type": "apache" }, "apache": { "status": { + "load": { + "5": 1.89, + "15": 1.07, + "1": 1.53 + }, + "total_accesses": 11, "connections": { "total": 0, "async": { + "closing": 0, "writing": 0, - "keep_alive": 0, - "closing": 0 + "keep_alive": 0 } }, - "total_bytes": 128, - "cpu": { - "children_user": 0, - "children_system": 0, - "load": 0.185185, - "user": 1.11, - "system": 1.79 - }, + "requests_per_sec": 0.916667, "scoreboard": { - "logging": 0, - "idle_cleanup": 0, "starting_up": 0, - "reading_request": 0, + "keepalive": 0, + "sending_reply": 1, + "logging": 0, + "gracefully_finishing": 0, "dns_lookup": 0, "closing_connection": 0, - "gracefully_finishing": 0, - "sending_reply": 1, - "keepalive": 0, - "total": 400, "open_slot": 325, - "waiting_for_connection": 74 + "total": 400, + "idle_cleanup": 0, + "waiting_for_connection": 74, + "reading_request": 0 }, + "bytes_per_sec": 0, + "bytes_per_request": 0, + "uptime": { + "server_uptime": 12, + "uptime": 12 + }, + "total_bytes": 0, "workers": { "busy": 1, "idle": 74 }, - "bytes_per_sec": 83.6986, - "uptime": { - "server_uptime": 1566, - "uptime": 1566 - }, - "total_accesses": 1393, - "bytes_per_request": 94.0933, - "requests_per_sec": 0.889527, - "load": { - "1": 3.58, - "5": 3.54, - "15": 2.79 + "cpu": { + "load": 0.583333, + "user": 0.03, + "system": 0.04, + "children_user": 0, + "children_system": 0 } } - }, - "service": { - "address": "127.0.0.1:8088", - "type": "apache" - }, - "event": { - "duration": 2381832, - "dataset": "apache.status", - "module": "apache" - }, - "ecs": { - "version": "1.5.0" - }, - "agent": { - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "685f03e4-76e7-4d05-b398-8454b8964681", - "id": "a74466da-3ea4-44f9-aea0-11c5e4b920be", - "name": "MacBook-Elastic.local" } } \ No newline at end of file diff --git a/test/packages/apache/docs/README.md b/test/packages/apache/docs/README.md index 34706dc901..d00e556d05 100644 --- a/test/packages/apache/docs/README.md +++ b/test/packages/apache/docs/README.md @@ -5,8 +5,8 @@ logs created by the Apache server. ## Compatibility -The Apache datasets were tested with Apache 2.4.12 and 2.4.20 and are expected to work with -all versions >= 2.2.31 and >= 2.4.16. +The Apache datasets were tested with Apache 2.4.12 and 2.4.46 and are expected to work with +all versions >= 2.2.31 and >= 2.4.16 (independent from operating system). ## Logs @@ -16,43 +16,94 @@ Access logs collects the Apache access logs. **Exported fields** -| Field | Description | Type | Unit | Metric Type | -|---|---|---|---|---| -| @timestamp | Event timestamp. | date | | | -| apache.access.ssl.cipher | SSL cipher name. | keyword | | | -| apache.access.ssl.protocol | SSL protocol version. | keyword | | | -| data_stream.dataset | Data stream dataset. | constant_keyword | | | -| data_stream.namespace | Data stream namespace. | constant_keyword | | | -| data_stream.type | Data stream type. | constant_keyword | | | -| ecs.version | | keyword | | | -| http.request.method | HTTP request method. Prior to ECS 1.6.0 the following guidance was provided: "The field value must be normalized to lowercase for querying." As of ECS 1.6.0, the guidance is deprecated because the original case of the method may be useful in anomaly detection. Original case will be mandated in ECS 2.0.0 | keyword | | | -| http.request.referrer | Referrer for this HTTP request. | keyword | | | -| http.response.body.bytes | Size in bytes of the response body. | long | byte | gauge | -| http.response.status_code | HTTP response status code. | long | | | -| http.version | HTTP version. | keyword | | | -| input.type | | keyword | | | -| log.file.path | | keyword | | | -| log.level | Original log level of the log event. If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity). Some examples are `warn`, `err`, `i`, `informational`. | keyword | | | -| log.offset | | long | | | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | text | | | -| process.pid | Process id. | long | | | -| process.thread.id | Thread ID. | long | | | -| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | | | -| source.geo.city_name | City name. | keyword | | | -| source.geo.continent_name | Name of the continent. | keyword | | | -| source.geo.country_iso_code | Country ISO code. | keyword | | | -| source.geo.location | Longitude and latitude. | geo_point | | | -| source.geo.region_iso_code | Region ISO code. | keyword | | | -| source.geo.region_name | Region name. | keyword | | | -| source.ip | | ip | | | -| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | keyword | | | -| user.name | Short name or login of the user. | keyword | | | -| user_agent.device.name | Name of the device. | keyword | | | -| user_agent.name | Name of the user agent. | keyword | | | -| user_agent.original | Unparsed user_agent string. | keyword | | | -| user_agent.os.name | Operating system name, without the version. | keyword | | | -| user_agent.os.version | Operating system version as a raw string. | keyword | | | -| user_agent.version | Version of the user agent. | keyword | | | +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| apache.access.ssl.cipher | SSL cipher name. | keyword | +| apache.access.ssl.protocol | SSL protocol version. | keyword | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| destination.domain | Destination domain. | keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| error.message | Error message. | match_only_text | +| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | +| event.created | event.created contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from @timestamp in that @timestamp typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, @timestamp should be used. | date | +| event.dataset | Event dataset | constant_keyword | +| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data coming in at a regular interval or not. | keyword | +| event.module | Event module | constant_keyword | +| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | +| file.path | Full path to the file, including the file name. It should include the drive letter, when appropriate. | keyword | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| http.request.method | HTTP request method. Prior to ECS 1.6.0 the following guidance was provided: "The field value must be normalized to lowercase for querying." As of ECS 1.6.0, the guidance is deprecated because the original case of the method may be useful in anomaly detection. Original case will be mandated in ECS 2.0.0 | keyword | +| http.request.referrer | Referrer for this HTTP request. | keyword | +| http.response.body.bytes | Size in bytes of the response body. | long | +| http.response.status_code | HTTP response status code. | long | +| http.version | HTTP version. | keyword | +| input.type | Input type | keyword | +| log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | +| log.level | Original log level of the log event. If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity). Some examples are `warn`, `err`, `i`, `informational`. | keyword | +| log.offset | Log offset | long | +| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | +| process.pid | Process id. | long | +| process.thread.id | Thread ID. | long | +| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | +| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | +| source.as.organization.name | Organization name. | keyword | +| source.domain | Source domain. | keyword | +| source.geo.city_name | City name. | keyword | +| source.geo.continent_name | Name of the continent. | keyword | +| source.geo.country_iso_code | Country ISO code. | keyword | +| source.geo.country_name | Country name. | keyword | +| source.geo.location | Longitude and latitude. | geo_point | +| source.geo.region_iso_code | Region ISO code. | keyword | +| source.geo.region_name | Region name. | keyword | +| source.ip | IP address of the source (IPv4 or IPv6). | ip | +| tags | List of keywords used to tag each event. | keyword | +| tls.cipher | String indicating the cipher used during the current connection. | keyword | +| tls.version | Numeric part of the version parsed from the original string. | keyword | +| tls.version_protocol | Normalized lowercase protocol name parsed from original string. | keyword | +| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | +| url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | +| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | +| url.path | Path of the request, such as "/search". | wildcard | +| url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | +| user.name | Short name or login of the user. | keyword | +| user_agent.device.name | Name of the device. | keyword | +| user_agent.name | Name of the user agent. | keyword | +| user_agent.original | Unparsed user_agent string. | keyword | +| user_agent.os.full | Operating system name, including the version or code name. | keyword | +| user_agent.os.name | Operating system name, without the version. | keyword | +| user_agent.os.version | Operating system version as a raw string. | keyword | +| user_agent.version | Version of the user agent. | keyword | ### Error Logs @@ -65,30 +116,77 @@ Error logs collects the Apache error logs. |---|---|---| | @timestamp | Event timestamp. | date | | apache.error.module | The module producing the logged message. | keyword | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | | keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| error.message | Error message. | match_only_text | +| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | +| event.dataset | Event dataset | constant_keyword | +| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data coming in at a regular interval or not. | keyword | +| event.module | Event module | constant_keyword | +| event.timezone | This field should be populated when the event's timestamp does not include timezone information already (e.g. default Syslog timestamps). It's optional otherwise. Acceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00"). | keyword | +| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | +| file.path | Full path to the file, including the file name. It should include the drive letter, when appropriate. | keyword | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | http.request.method | HTTP request method. Prior to ECS 1.6.0 the following guidance was provided: "The field value must be normalized to lowercase for querying." As of ECS 1.6.0, the guidance is deprecated because the original case of the method may be useful in anomaly detection. Original case will be mandated in ECS 2.0.0 | keyword | | http.request.referrer | Referrer for this HTTP request. | keyword | | http.response.body.bytes | Size in bytes of the response body. | long | | http.response.status_code | HTTP response status code. | long | | http.version | HTTP version. | keyword | -| input.type | | keyword | -| log.file.path | | keyword | +| input.type | Input type | keyword | +| log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | | log.level | Original log level of the log event. If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity). Some examples are `warn`, `err`, `i`, `informational`. | keyword | -| log.offset | | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | text | +| log.offset | Log offset | long | +| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | | process.pid | Process id. | long | | process.thread.id | Thread ID. | long | | source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | +| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | +| source.as.organization.name | Organization name. | keyword | | source.geo.city_name | City name. | keyword | | source.geo.continent_name | Name of the continent. | keyword | | source.geo.country_iso_code | Country ISO code. | keyword | +| source.geo.country_name | Country name. | keyword | | source.geo.location | Longitude and latitude. | geo_point | | source.geo.region_iso_code | Region ISO code. | keyword | | source.geo.region_name | Region name. | keyword | -| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | keyword | +| source.ip | IP address of the source (IPv4 or IPv6). | ip | +| source.port | Port of the source. | long | +| tags | List of keywords used to tag each event. | keyword | +| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | +| url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | +| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | +| url.path | Path of the request, such as "/search". | wildcard | +| url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | | user.name | Short name or login of the user. | keyword | | user_agent.device.name | Name of the device. | keyword | | user_agent.name | Name of the user agent. | keyword | @@ -107,125 +205,182 @@ An example event for `status` looks as following: ```json { - "@timestamp": "2020-06-24T10:19:48.005Z", + "@timestamp": "2020-12-03T16:31:04.445Z", + "data_stream": { + "type": "metrics", + "dataset": "apache.status", + "namespace": "ep" + }, + "elastic_agent": { + "version": "7.11.0", + "id": "6c69e2bc-7bb3-4bac-b7e9-41f22558321c", + "snapshot": true + }, + "host": { + "os": { + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.9.184-linuxkit", + "codename": "Core" + }, + "id": "06c26569966fd125c15acac5d7feffb6", + "name": "4942ef7a8cfc", + "containerized": true, + "ip": [ + "192.168.0.4" + ], + "mac": [ + "02:42:c0:a8:00:04" + ], + "hostname": "4942ef7a8cfc", + "architecture": "x86_64" + }, + "agent": { + "hostname": "4942ef7a8cfc", + "ephemeral_id": "8371d3a3-5321-4436-9fd5-cafcabfe4c57", + "id": "af6f66ef-d7d0-4784-b9bb-3fddbcc151b5", + "name": "4942ef7a8cfc", + "type": "metricbeat", + "version": "7.11.0" + }, "metricset": { "name": "status", - "period": 10000 + "period": 30000 + }, + "service": { + "address": "http://elastic-package-service_apache_1:80/server-status?auto=", + "type": "apache" }, "apache": { "status": { + "load": { + "5": 1.89, + "15": 1.07, + "1": 1.53 + }, + "total_accesses": 11, "connections": { "total": 0, "async": { + "closing": 0, "writing": 0, - "keep_alive": 0, - "closing": 0 + "keep_alive": 0 } }, - "total_bytes": 128, - "cpu": { - "children_user": 0, - "children_system": 0, - "load": 0.185185, - "user": 1.11, - "system": 1.79 - }, + "requests_per_sec": 0.916667, "scoreboard": { - "logging": 0, - "idle_cleanup": 0, "starting_up": 0, - "reading_request": 0, + "keepalive": 0, + "sending_reply": 1, + "logging": 0, + "gracefully_finishing": 0, "dns_lookup": 0, "closing_connection": 0, - "gracefully_finishing": 0, - "sending_reply": 1, - "keepalive": 0, - "total": 400, "open_slot": 325, - "waiting_for_connection": 74 + "total": 400, + "idle_cleanup": 0, + "waiting_for_connection": 74, + "reading_request": 0 + }, + "bytes_per_sec": 0, + "bytes_per_request": 0, + "uptime": { + "server_uptime": 12, + "uptime": 12 }, + "total_bytes": 0, "workers": { "busy": 1, "idle": 74 }, - "bytes_per_sec": 83.6986, - "uptime": { - "server_uptime": 1566, - "uptime": 1566 - }, - "total_accesses": 1393, - "bytes_per_request": 94.0933, - "requests_per_sec": 0.889527, - "load": { - "1": 3.58, - "5": 3.54, - "15": 2.79 + "cpu": { + "load": 0.583333, + "user": 0.03, + "system": 0.04, + "children_user": 0, + "children_system": 0 } } - }, - "service": { - "address": "127.0.0.1:8088", - "type": "apache" - }, - "event": { - "duration": 2381832, - "dataset": "apache.status", - "module": "apache" - }, - "ecs": { - "version": "1.5.0" - }, - "agent": { - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "685f03e4-76e7-4d05-b398-8454b8964681", - "id": "a74466da-3ea4-44f9-aea0-11c5e4b920be", - "name": "MacBook-Elastic.local" } } ``` **Exported fields** -| Field | Description | Type | -|---|---|---| -| @timestamp | Event timestamp. | date | -| apache.status.bytes_per_request | Bytes per request. | scaled_float | -| apache.status.bytes_per_sec | Bytes per second. | scaled_float | -| apache.status.connections.async.closing | Async closed connections. | long | -| apache.status.connections.async.keep_alive | Async keeped alive connections. | long | -| apache.status.connections.async.writing | Async connection writing. | long | -| apache.status.connections.total | Total connections. | long | -| apache.status.cpu.children_system | CPU of children system. | scaled_float | -| apache.status.cpu.children_user | CPU of children user. | scaled_float | -| apache.status.cpu.load | CPU Load. | scaled_float | -| apache.status.cpu.system | System cpu. | scaled_float | -| apache.status.cpu.user | CPU user load. | scaled_float | -| apache.status.load.1 | Load average for the last minute. | scaled_float | -| apache.status.load.15 | Load average for the last 15 minutes. | scaled_float | -| apache.status.load.5 | Load average for the last 5 minutes. | scaled_float | -| apache.status.requests_per_sec | Requests per second. | scaled_float | -| apache.status.scoreboard.closing_connection | Closing connections. | long | -| apache.status.scoreboard.dns_lookup | Dns Lookups. | long | -| apache.status.scoreboard.gracefully_finishing | Gracefully finishing. | long | -| apache.status.scoreboard.idle_cleanup | Idle cleanups. | long | -| apache.status.scoreboard.keepalive | Keep alive. | long | -| apache.status.scoreboard.logging | Logging | long | -| apache.status.scoreboard.open_slot | Open slots. | long | -| apache.status.scoreboard.reading_request | Reading requests. | long | -| apache.status.scoreboard.sending_reply | Sending Reply. | long | -| apache.status.scoreboard.starting_up | Starting up. | long | -| apache.status.scoreboard.total | Total. | long | -| apache.status.scoreboard.waiting_for_connection | Waiting for connections. | long | -| apache.status.total_accesses | Total number of access requests. | long | -| apache.status.total_bytes | Total number of bytes served. | long | -| apache.status.uptime.server_uptime | Server uptime in seconds. | long | -| apache.status.uptime.uptime | Server uptime. | long | -| apache.status.workers.busy | Number of busy workers. | long | -| apache.status.workers.idle | Number of idle workers. | long | -| data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | Data stream namespace. | constant_keyword | -| data_stream.type | Data stream type. | constant_keyword | -| ecs.version | | keyword | -| service.address | | keyword | -| service.type | | keyword | +| Field | Description | Type | Unit | Metric Type | +|---|---|---|---|---| +| @timestamp | Event timestamp. | date | | | +| apache.status.bytes_per_request | Bytes per request. | scaled_float | | gauge | +| apache.status.bytes_per_sec | Bytes per second. | scaled_float | | gauge | +| apache.status.connections.async.closing | Async closed connections. | long | | gauge | +| apache.status.connections.async.keep_alive | Async keeped alive connections. | long | | gauge | +| apache.status.connections.async.writing | Async connection writing. | long | | gauge | +| apache.status.connections.total | Total connections. | long | | counter | +| apache.status.cpu.children_system | CPU of children system. | scaled_float | | gauge | +| apache.status.cpu.children_user | CPU of children user. | scaled_float | | gauge | +| apache.status.cpu.load | CPU Load. | scaled_float | | gauge | +| apache.status.cpu.system | System cpu. | scaled_float | | gauge | +| apache.status.cpu.user | CPU user load. | scaled_float | | gauge | +| apache.status.load.1 | Load average for the last minute. | scaled_float | | gauge | +| apache.status.load.15 | Load average for the last 15 minutes. | scaled_float | | gauge | +| apache.status.load.5 | Load average for the last 5 minutes. | scaled_float | | gauge | +| apache.status.requests_per_sec | Requests per second. | scaled_float | | gauge | +| apache.status.scoreboard.closing_connection | Closing connections. | long | | gauge | +| apache.status.scoreboard.dns_lookup | Dns Lookups. | long | | gauge | +| apache.status.scoreboard.gracefully_finishing | Gracefully finishing. | long | | gauge | +| apache.status.scoreboard.idle_cleanup | Idle cleanups. | long | | gauge | +| apache.status.scoreboard.keepalive | Keep alive. | long | | gauge | +| apache.status.scoreboard.logging | Logging | long | | gauge | +| apache.status.scoreboard.open_slot | Open slots. | long | | gauge | +| apache.status.scoreboard.reading_request | Reading requests. | long | | gauge | +| apache.status.scoreboard.sending_reply | Sending Reply. | long | | gauge | +| apache.status.scoreboard.starting_up | Starting up. | long | | gauge | +| apache.status.scoreboard.total | Total. | long | | gauge | +| apache.status.scoreboard.waiting_for_connection | Waiting for connections. | long | | gauge | +| apache.status.total_accesses | Total number of access requests. | long | | counter | +| apache.status.total_bytes | Total number of bytes served. | long | byte | counter | +| apache.status.uptime.server_uptime | Server uptime in seconds. | long | | counter | +| apache.status.uptime.uptime | Server uptime. | long | | counter | +| apache.status.workers.busy | Number of busy workers. | long | | gauge | +| apache.status.workers.idle | Number of idle workers. | long | | gauge | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | | | +| cloud.image.id | Image ID for the cloud instance. | keyword | | | +| cloud.instance.id | Instance ID of the host machine. | keyword | | | +| cloud.instance.name | Instance name of the host machine. | keyword | | | +| cloud.machine.type | Machine type of the host machine. | keyword | | | +| cloud.project.id | Name of the project in Google Cloud. | keyword | | | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | | | +| cloud.region | Region in which this host is running. | keyword | | | +| container.id | Unique container id. | keyword | | | +| container.image.name | Name of the image the container was built on. | keyword | | | +| container.labels | Image labels. | object | | | +| container.name | Container name. | keyword | | | +| data_stream.dataset | Data stream dataset. | constant_keyword | | | +| data_stream.namespace | Data stream namespace. | constant_keyword | | | +| data_stream.type | Data stream type. | constant_keyword | | | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | | +| error.message | Error message. | match_only_text | | | +| event.dataset | Event dataset | constant_keyword | | | +| event.module | Event module | constant_keyword | | | +| host.architecture | Operating system architecture. | keyword | | | +| host.containerized | If the host is a container. | boolean | | | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | | | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | | | +| host.ip | Host ip addresses. | ip | | | +| host.mac | Host mac addresses. | keyword | | | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | | | +| host.os.build | OS build information. | keyword | | | +| host.os.codename | OS codename, if any. | keyword | | | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | | | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | | | +| host.os.name | Operating system name, without the version. | keyword | | | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | | +| host.os.version | Operating system version as a raw string. | keyword | | | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | | +| service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | | + diff --git a/test/packages/apache/img/apache-logs-overview.png b/test/packages/apache/img/apache-logs-overview.png new file mode 100644 index 0000000000..5597f61a27 Binary files /dev/null and b/test/packages/apache/img/apache-logs-overview.png differ diff --git a/test/packages/apache/img/apache-metrics-overview.png b/test/packages/apache/img/apache-metrics-overview.png new file mode 100644 index 0000000000..ec24030bbd Binary files /dev/null and b/test/packages/apache/img/apache-metrics-overview.png differ diff --git a/test/packages/apache/img/apache_httpd_server_status.png b/test/packages/apache/img/apache_httpd_server_status.png deleted file mode 100644 index b28bbecb34..0000000000 Binary files a/test/packages/apache/img/apache_httpd_server_status.png and /dev/null differ diff --git a/test/packages/apache/img/kibana-apache.png b/test/packages/apache/img/kibana-apache.png deleted file mode 100644 index badfee933a..0000000000 Binary files a/test/packages/apache/img/kibana-apache.png and /dev/null differ diff --git a/test/packages/apache/kibana/dashboard/apache-Logs-Apache-Dashboard.json b/test/packages/apache/kibana/dashboard/apache-Logs-Apache-Dashboard.json index 6cf77308fd..7332f02b4d 100644 --- a/test/packages/apache/kibana/dashboard/apache-Logs-Apache-Dashboard.json +++ b/test/packages/apache/kibana/dashboard/apache-Logs-Apache-Dashboard.json @@ -14,11 +14,13 @@ } }, "optionsJSON": { - "darkTheme": false + "darkTheme": false, + "useMargins": true }, "panelsJSON": [ { "embeddableConfig": { + "enhancements": {}, "mapBounds": { "bottom_right": { "lat": -3.864254615721396, @@ -49,16 +51,18 @@ "gridData": { "h": 12, "i": "1", - "w": 48, - "x": 0, + "w": 35, + "x": 13, "y": 0 }, "panelIndex": "1", "panelRefName": "panel_0", - "version": "7.3.0" + "version": "7.9.3" }, { - "embeddableConfig": {}, + "embeddableConfig": { + "enhancements": {} + }, "gridData": { "h": 12, "i": "2", @@ -68,10 +72,12 @@ }, "panelIndex": "2", "panelRefName": "panel_1", - "version": "7.3.0" + "version": "7.9.3" }, { - "embeddableConfig": {}, + "embeddableConfig": { + "enhancements": {} + }, "gridData": { "h": 12, "i": "3", @@ -81,10 +87,12 @@ }, "panelIndex": "3", "panelRefName": "panel_2", - "version": "7.3.0" + "version": "7.9.3" }, { - "embeddableConfig": {}, + "embeddableConfig": { + "enhancements": {} + }, "gridData": { "h": 8, "i": "4", @@ -94,10 +102,12 @@ }, "panelIndex": "4", "panelRefName": "panel_3", - "version": "7.3.0" + "version": "7.9.3" }, { - "embeddableConfig": {}, + "embeddableConfig": { + "enhancements": {} + }, "gridData": { "h": 8, "i": "5", @@ -107,10 +117,12 @@ }, "panelIndex": "5", "panelRefName": "panel_4", - "version": "7.3.0" + "version": "7.9.3" }, { - "embeddableConfig": {}, + "embeddableConfig": { + "enhancements": {} + }, "gridData": { "h": 8, "i": "6", @@ -120,7 +132,7 @@ }, "panelIndex": "6", "panelRefName": "panel_5", - "version": "7.3.0" + "version": "7.9.3" }, { "embeddableConfig": { @@ -130,6 +142,7 @@ "apache2.error.integration", "message" ], + "enhancements": {}, "sort": [ "@timestamp", "desc" @@ -144,7 +157,22 @@ }, "panelIndex": "7", "panelRefName": "panel_6", - "version": "7.3.0" + "version": "7.9.3" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 5, + "i": "f5d1286d-411a-4759-a2e2-0b3227b93cfa", + "w": 13, + "x": 0, + "y": 0 + }, + "panelIndex": "f5d1286d-411a-4759-a2e2-0b3227b93cfa", + "panelRefName": "panel_7", + "version": "7.9.3" } ], "timeRestore": false, @@ -153,7 +181,7 @@ }, "id": "apache-Logs-Apache-Dashboard", "migrationVersion": { - "dashboard": "7.3.0" + "dashboard": "7.11.0" }, "references": [ { @@ -190,6 +218,11 @@ "id": "apache-errors-log", "name": "panel_6", "type": "search" + }, + { + "id": "apache-ed44f820-3a10-11eb-8946-296aab7b13db", + "name": "panel_7", + "type": "visualization" } ], "type": "dashboard" diff --git a/test/packages/apache/kibana/dashboard/apache-Metrics-Apache-HTTPD-server-status-2.json b/test/packages/apache/kibana/dashboard/apache-Metrics-Apache-HTTPD-server-status-2.json deleted file mode 100644 index 83f1d1551c..0000000000 --- a/test/packages/apache/kibana/dashboard/apache-Metrics-Apache-HTTPD-server-status-2.json +++ /dev/null @@ -1,179 +0,0 @@ -{ - "attributes": { - "description": "Overview of Apache server status", - "hits": 0, - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [], - "highlightAll": true, - "query": { - "language": "kuery", - "query": "" - }, - "version": true - } - }, - "optionsJSON": { - "darkTheme": false - }, - "panelsJSON": [ - { - "embeddableConfig": {}, - "gridData": { - "h": 12, - "i": "1", - "w": 24, - "x": 24, - "y": 36 - }, - "panelIndex": "1", - "panelRefName": "panel_0", - "version": "7.3.0" - }, - { - "embeddableConfig": { - "vis": { - "params": { - "sort": { - "columnIndex": null, - "direction": null - } - } - } - }, - "gridData": { - "h": 12, - "i": "2", - "w": 12, - "x": 0, - "y": 0 - }, - "panelIndex": "2", - "panelRefName": "panel_1", - "version": "7.3.0" - }, - { - "embeddableConfig": {}, - "gridData": { - "h": 12, - "i": "3", - "w": 24, - "x": 0, - "y": 36 - }, - "panelIndex": "3", - "panelRefName": "panel_2", - "version": "7.3.0" - }, - { - "embeddableConfig": {}, - "gridData": { - "h": 12, - "i": "4", - "w": 48, - "x": 0, - "y": 24 - }, - "panelIndex": "4", - "panelRefName": "panel_3", - "version": "7.3.0" - }, - { - "embeddableConfig": { - "vis": { - "defaultColors": { - "0 - 100": "rgb(0,104,55)" - } - } - }, - "gridData": { - "h": 12, - "i": "5", - "w": 24, - "x": 24, - "y": 0 - }, - "panelIndex": "5", - "panelRefName": "panel_4", - "version": "7.3.0" - }, - { - "embeddableConfig": { - "vis": { - "defaultColors": { - "0 - 100": "rgb(0,104,55)" - } - } - }, - "gridData": { - "h": 12, - "i": "6", - "w": 12, - "x": 12, - "y": 0 - }, - "panelIndex": "6", - "panelRefName": "panel_5", - "version": "7.3.0" - }, - { - "embeddableConfig": {}, - "gridData": { - "h": 12, - "i": "7", - "w": 48, - "x": 0, - "y": 12 - }, - "panelIndex": "7", - "panelRefName": "panel_6", - "version": "7.3.0" - } - ], - "timeRestore": false, - "title": "[Metrics Apache] Overview", - "version": 1 - }, - "id": "apache-Metrics-Apache-HTTPD-server-status-2", - "migrationVersion": { - "dashboard": "7.3.0" - }, - "references": [ - { - "id": "apache-HTTPD-CPU", - "name": "panel_0", - "type": "visualization" - }, - { - "id": "apache-HTTPD-Hostname-list", - "name": "panel_1", - "type": "visualization" - }, - { - "id": "apache-HTTPD-Load1-slash-5-slash-15", - "name": "panel_2", - "type": "visualization" - }, - { - "id": "apache-HTTPD-Scoreboard", - "name": "panel_3", - "type": "visualization" - }, - { - "id": "apache-HTTPD-Total-accesses-and-kbytes", - "name": "panel_4", - "type": "visualization" - }, - { - "id": "apache-HTTPD-Uptime", - "name": "panel_5", - "type": "visualization" - }, - { - "id": "apache-HTTPD-Workers", - "name": "panel_6", - "type": "visualization" - } - ], - "type": "dashboard" -} \ No newline at end of file diff --git a/test/packages/apache/kibana/dashboard/apache-Metrics-Apache-HTTPD-server-status.json b/test/packages/apache/kibana/dashboard/apache-Metrics-Apache-HTTPD-server-status.json new file mode 100644 index 0000000000..35a2998fd1 --- /dev/null +++ b/test/packages/apache/kibana/dashboard/apache-Metrics-Apache-HTTPD-server-status.json @@ -0,0 +1,273 @@ +{ + "attributes": { + "description": "Overview of Apache server status", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "highlightAll": true, + "query": { + "language": "kuery", + "query": "" + }, + "version": true + } + }, + "optionsJSON": { + "darkTheme": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 15, + "i": "1", + "w": 24, + "x": 24, + "y": 50 + }, + "panelIndex": "1", + "panelRefName": "panel_0", + "version": "7.9.3" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 15, + "i": "3", + "w": 24, + "x": 0, + "y": 50 + }, + "panelIndex": "3", + "panelRefName": "panel_1", + "version": "7.9.3" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 15, + "i": "4", + "w": 24, + "x": 24, + "y": 5 + }, + "panelIndex": "4", + "panelRefName": "panel_2", + "version": "7.9.3" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 5, + "i": "4cc4755f-49a7-43c1-8a21-0a78291f0b3f", + "w": 13, + "x": 0, + "y": 0 + }, + "panelIndex": "4cc4755f-49a7-43c1-8a21-0a78291f0b3f", + "panelRefName": "panel_3", + "version": "7.9.3" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 5, + "i": "7b7a1f18-e274-4f4e-a3b3-3760e7896897", + "w": 11, + "x": 13, + "y": 0 + }, + "panelIndex": "7b7a1f18-e274-4f4e-a3b3-3760e7896897", + "panelRefName": "panel_4", + "version": "7.9.3" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 5, + "i": "01794c9e-0ce3-4e1e-bc87-6c15b6434ba8", + "w": 12, + "x": 24, + "y": 0 + }, + "panelIndex": "01794c9e-0ce3-4e1e-bc87-6c15b6434ba8", + "panelRefName": "panel_5", + "version": "7.9.3" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 5, + "i": "e2b2dd17-dcda-4b17-b250-dd30c596f7f6", + "w": 12, + "x": 36, + "y": 0 + }, + "panelIndex": "e2b2dd17-dcda-4b17-b250-dd30c596f7f6", + "panelRefName": "panel_6", + "version": "7.9.3" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 15, + "i": "f90d54d8-034c-4cfd-8640-0e8f10c2ca99", + "w": 24, + "x": 0, + "y": 5 + }, + "panelIndex": "f90d54d8-034c-4cfd-8640-0e8f10c2ca99", + "panelRefName": "panel_7", + "version": "7.9.3" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 15, + "i": "3b23e577-34f6-4cd0-b636-01581b8ce1c0", + "w": 24, + "x": 0, + "y": 20 + }, + "panelIndex": "3b23e577-34f6-4cd0-b636-01581b8ce1c0", + "panelRefName": "panel_8", + "version": "7.9.3" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 15, + "i": "100b598b-4223-4a6b-95d9-ee94147fa5ac", + "w": 24, + "x": 24, + "y": 20 + }, + "panelIndex": "100b598b-4223-4a6b-95d9-ee94147fa5ac", + "panelRefName": "panel_9", + "version": "7.9.3" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 15, + "i": "a7dc6253-4f39-4aae-984f-3108d1cf3cfb", + "w": 24, + "x": 0, + "y": 35 + }, + "panelIndex": "a7dc6253-4f39-4aae-984f-3108d1cf3cfb", + "panelRefName": "panel_10", + "version": "7.9.3" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 15, + "i": "4204480f-8c9f-426f-b3f7-0714a70d418b", + "w": 24, + "x": 24, + "y": 35 + }, + "panelIndex": "4204480f-8c9f-426f-b3f7-0714a70d418b", + "panelRefName": "panel_11", + "version": "7.9.3" + } + ], + "timeRestore": false, + "title": "[Metrics Apache] Overview", + "version": 1 + }, + "id": "apache-Metrics-Apache-HTTPD-server-status", + "migrationVersion": { + "dashboard": "7.11.0" + }, + "references": [ + { + "id": "apache-HTTPD-CPU", + "name": "panel_0", + "type": "visualization" + }, + { + "id": "apache-HTTPD-Load1-slash-5-slash-15", + "name": "panel_1", + "type": "visualization" + }, + { + "id": "apache-HTTPD-Scoreboard", + "name": "panel_2", + "type": "visualization" + }, + { + "id": "apache-805d7bb0-3a10-11eb-8946-296aab7b13db", + "name": "panel_3", + "type": "visualization" + }, + { + "id": "apache-22057f20-3a12-11eb-8946-296aab7b13db", + "name": "panel_4", + "type": "visualization" + }, + { + "id": "apache-47820ce0-3a1d-11eb-8946-296aab7b13db", + "name": "panel_5", + "type": "visualization" + }, + { + "id": "apache-99666080-3a20-11eb-8946-296aab7b13db", + "name": "panel_6", + "type": "visualization" + }, + { + "id": "apache-f4ffec70-3a36-11eb-8946-296aab7b13db", + "name": "panel_7", + "type": "visualization" + }, + { + "id": "apache-320cd980-3a36-11eb-8946-296aab7b13db", + "name": "panel_8", + "type": "visualization" + }, + { + "id": "apache-a45311f0-3a34-11eb-8946-296aab7b13db", + "name": "panel_9", + "type": "visualization" + }, + { + "id": "apache-7d68f730-3a39-11eb-8946-296aab7b13db", + "name": "panel_10", + "type": "visualization" + }, + { + "id": "apache-7724cf20-3a39-11eb-8946-296aab7b13db", + "name": "panel_11", + "type": "visualization" + } + ], + "type": "dashboard" +} \ No newline at end of file diff --git a/test/packages/apache/kibana/ml_module/apache-Logs-ml.json b/test/packages/apache/kibana/ml_module/apache-Logs-ml.json new file mode 100644 index 0000000000..cccc306d3f --- /dev/null +++ b/test/packages/apache/kibana/ml_module/apache-Logs-ml.json @@ -0,0 +1,419 @@ +{ + "attributes": { + "id": "apache_data_stream", + "title": "Apache access logs", + "description": "Find unusual activity in HTTP access logs.", + "type": "Web Access Logs", + "logo": { + "icon": "logoApache" + }, + "defaultIndexPattern": "logs-*", + "query": { + "bool": { + "filter": [ + { + "term": { + "data_stream.dataset": "apache.access" + } + }, + { + "exists": { + "field": "source.address" + } + }, + { + "exists": { + "field": "url.original" + } + }, + { + "exists": { + "field": "http.response.status_code" + } + } + ], + "must_not": { + "terms": { + "_tier": [ + "data_frozen", + "data_cold" + ] + } + } + } + }, + "jobs": [ + { + "id": "visitor_rate_apache", + "config": { + "groups": [ + "apache" + ], + "description": "HTTP Access Logs: Detect unusual visitor rates", + "analysis_config": { + "bucket_span": "15m", + "summary_count_field_name": "dc_source_address", + "detectors": [ + { + "detector_description": "Apache access visitor rate", + "function": "non_zero_count" + } + ], + "influencers": [] + }, + "analysis_limits": { + "model_memory_limit": "10mb" + }, + "data_description": { + "time_field": "@timestamp", + "time_format": "epoch_ms" + }, + "model_plot_config": { + "enabled": true + }, + "custom_settings": { + "created_by": "ml-module-apache-access-data-stream", + "custom_urls": [ + { + "url_name": "Apache logs overview", + "url_value": "dashboards#/view/apache-Logs-Apache-Dashboard?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))\u0026_a=(description:\u0027\u0027,filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:data_stream.dataset,negate:!f,params:(query:\u0027apache.access\u0027),type:phrase,value:\u0027apache.access\u0027),query:(match:(data_stream.dataset:(query:\u0027apache.access\u0027,type:phrase))))),query:(language:kuery,query:\u0027\u0027))" + }, + { + "url_name": "Raw data", + "url_value": "discover#/?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))\u0026_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:data_stream.dataset,negate:!f,params:(query:\u0027apache.access\u0027),type:phrase,value:\u0027apache.access\u0027),query:(match:(data_stream.dataset:(query:\u0027apache.access\u0027,type:phrase))))),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:\u0027\u0027),sort:!(\u0027@timestamp\u0027,desc))" + } + ] + } + } + }, + { + "id": "status_code_rate_apache", + "config": { + "groups": [ + "apache" + ], + "description": "HTTP Access Logs: Detect unusual status code rates", + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "Apache access status code rate", + "function": "count", + "partition_field_name": "http.response.status_code" + } + ], + "influencers": [ + "http.response.status_code", + "source.address" + ] + }, + "analysis_limits": { + "model_memory_limit": "100mb" + }, + "data_description": { + "time_field": "@timestamp", + "time_format": "epoch_ms" + }, + "model_plot_config": { + "enabled": true + }, + "custom_settings": { + "created_by": "ml-module-apache-access-data-stream", + "custom_urls": [ + { + "url_name": "Apache logs overview", + "url_value": "dashboards#/view/apache-Logs-Apache-Dashboard?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))\u0026_a=(description:\u0027\u0027,filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:data_stream.dataset,negate:!f,params:(query:\u0027apache.access\u0027),type:phrase,value:\u0027apache.access\u0027),query:(match:(data_stream.dataset:(query:\u0027apache.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:http.response.status_code,negate:!f,params:(query:\u0027$http.response.status_code$\u0027),type:phrase,value:\u0027$http.response.status_code$\u0027),query:(match:(http.response.status_code:(query:\u0027$http.response.status_code$\u0027,type:phrase))))),query:(language:kuery,query:\u0027\u0027))" + }, + { + "url_name": "Raw data", + "url_value": "discover#/?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))\u0026_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:data_stream.dataset,negate:!f,params:(query:\u0027apache.access\u0027),type:phrase,value:\u0027apache.access\u0027),query:(match:(data_stream.dataset:(query:\u0027apache.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:http.response.status_code,negate:!f,params:(query:\u0027$http.response.status_code$\u0027),type:phrase,value:\u0027$http.response.status_code$\u0027),query:(match:(http.response.status_code:(query:\u0027$http.response.status_code$\u0027,type:phrase))))),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:\u0027\u0027),sort:!(\u0027@timestamp\u0027,desc))" + } + ] + } + } + }, + { + "id": "source_ip_url_count_apache", + "config": { + "groups": [ + "apache" + ], + "description": "HTTP Access Logs: Detect unusual source IPs - high distinct count of URLs", + "analysis_config": { + "bucket_span": "1h", + "detectors": [ + { + "detector_description": "Apache access source IP high dc URL", + "function": "high_distinct_count", + "field_name": "url.original", + "over_field_name": "source.address" + } + ], + "influencers": [ + "source.address" + ] + }, + "data_description": { + "time_field": "@timestamp", + "time_format": "epoch_ms" + }, + "custom_settings": { + "created_by": "ml-module-apache-access-data-stream", + "custom_urls": [ + { + "url_name": "Apache logs overview", + "url_value": "dashboards#/view/apache-Logs-Apache-Dashboard?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))\u0026_a=(description:\u0027\u0027,filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:data_stream.dataset,negate:!f,params:(query:\u0027apache.access\u0027),type:phrase,value:\u0027apache.access\u0027),query:(match:(data_stream.dataset:(query:\u0027apache.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:source.address,negate:!f,params:(query:\u0027$source.address$\u0027),type:phrase,value:\u0027$source.address$\u0027),query:(match:(source.address:(query:\u0027$source.address$\u0027,type:phrase))))),query:(language:kuery,query:\u0027\u0027))" + }, + { + "url_name": "Raw data", + "url_value": "discover#/?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))\u0026_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:data_stream.dataset,negate:!f,params:(query:\u0027apache.access\u0027),type:phrase,value:\u0027apache.access\u0027),query:(match:(data_stream.dataset:(query:\u0027apache.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:source.address,negate:!f,params:(query:\u0027$source.address$\u0027),type:phrase,value:\u0027$source.address$\u0027),query:(match:(source.address:(query:\u0027$source.address$\u0027,type:phrase))))),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:\u0027\u0027),sort:!(\u0027@timestamp\u0027,desc))" + } + ] + } + } + }, + { + "id": "source_ip_request_rate_apache", + "config": { + "groups": [ + "apache" + ], + "description": "HTTP Access Logs: Detect unusual source IPs - high request rates", + "analysis_config": { + "bucket_span": "1h", + "detectors": [ + { + "detector_description": "Apache access source IP high count", + "function": "high_count", + "over_field_name": "source.address" + } + ], + "influencers": [ + "source.address" + ] + }, + "data_description": { + "time_field": "@timestamp", + "time_format": "epoch_ms" + }, + "custom_settings": { + "created_by": "ml-module-apache-access-data-stream", + "custom_urls": [ + { + "url_name": "Apache logs overview", + "url_value": "dashboards#/view/apache-Logs-Apache-Dashboard?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))\u0026_a=(description:\u0027\u0027,filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:data_stream.dataset,negate:!f,params:(query:\u0027apache.access\u0027),type:phrase,value:\u0027apache.access\u0027),query:(match:(data_stream.dataset:(query:\u0027apache.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:source.address,negate:!f,params:(query:\u0027$source.address$\u0027),type:phrase,value:\u0027$source.address$\u0027),query:(match:(source.address:(query:\u0027$source.address$\u0027,type:phrase))))),query:(language:kuery,query:\u0027\u0027))" + }, + { + "url_name": "Raw data", + "url_value": "discover#/?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))\u0026_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:data_stream.dataset,negate:!f,params:(query:\u0027apache.access\u0027),type:phrase,value:\u0027apache.access\u0027),query:(match:(data_stream.dataset:(query:\u0027apache.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:source.address,negate:!f,params:(query:\u0027$source.address$\u0027),type:phrase,value:\u0027$source.address$\u0027),query:(match:(source.address:(query:\u0027$source.address$\u0027,type:phrase))))),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:\u0027\u0027),sort:!(\u0027@timestamp\u0027,desc))" + } + ] + } + } + }, + { + "id": "low_request_rate_apache", + "config": { + "groups": [ + "apache" + ], + "description": "HTTP Access Logs: Detect low request rates", + "analysis_config": { + "bucket_span": "15m", + "summary_count_field_name": "doc_count", + "detectors": [ + { + "detector_description": "Apache access low request rate", + "function": "low_count" + } + ], + "influencers": [] + }, + "analysis_limits": { + "model_memory_limit": "10mb" + }, + "data_description": { + "time_field": "@timestamp", + "time_format": "epoch_ms" + }, + "model_plot_config": { + "enabled": true + }, + "custom_settings": { + "created_by": "ml-module-apache-access-data-stream", + "custom_urls": [ + { + "url_name": "Apache logs overview", + "url_value": "dashboards#/view/apache-Logs-Apache-Dashboard?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))\u0026_a=(description:\u0027\u0027,filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:data_stream.dataset,negate:!f,params:(query:\u0027apache.access\u0027),type:phrase,value:\u0027apache.access\u0027),query:(match:(data_stream.dataset:(query:\u0027apache.access\u0027,type:phrase))))),query:(language:kuery,query:\u0027\u0027))" + }, + { + "url_name": "Raw data", + "url_value": "discover#/?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))\u0026_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:data_stream.dataset,negate:!f,params:(query:\u0027apache.access\u0027),type:phrase,value:\u0027apache.access\u0027),query:(match:(data_stream.dataset:(query:\u0027apache.access\u0027,type:phrase))))),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:\u0027\u0027),sort:!(\u0027@timestamp\u0027,desc))" + } + ] + } + } + } + ], + "datafeeds": [ + { + "id": "datafeed-visitor_rate_apache", + "job_id": "visitor_rate_apache", + "config": { + "job_id": "visitor_rate_apache", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "query": { + "bool": { + "filter": [ + { + "term": { + "data_stream.dataset": "apache.access" + } + } + ] + } + }, + "aggregations": { + "buckets": { + "date_histogram": { + "field": "@timestamp", + "fixed_interval": "15m", + "offset": 0, + "order": { + "_key": "asc" + }, + "keyed": false, + "min_doc_count": 0 + }, + "aggregations": { + "@timestamp": { + "max": { + "field": "@timestamp" + } + }, + "dc_source_address": { + "cardinality": { + "field": "source.address" + } + } + } + } + } + } + }, + { + "id": "datafeed-status_code_rate_apache", + "job_id": "status_code_rate_apache", + "config": { + "job_id": "status_code_rate_apache", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "query": { + "bool": { + "filter": [ + { + "term": { + "data_stream.dataset": "apache.access" + } + } + ] + } + } + } + }, + { + "id": "datafeed-source_ip_url_count_apache", + "job_id": "source_ip_url_count_apache", + "config": { + "job_id": "source_ip_url_count_apache", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "query": { + "bool": { + "filter": [ + { + "term": { + "data_stream.dataset": "apache.access" + } + } + ] + } + } + } + }, + { + "id": "datafeed-source_ip_request_rate_apache", + "job_id": "source_ip_request_rate_apache", + "config": { + "job_id": "source_ip_request_rate_apache", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "query": { + "bool": { + "filter": [ + { + "term": { + "data_stream.dataset": "apache.access" + } + } + ] + } + } + } + }, + { + "id": "datafeed-low_request_rate_apache", + "job_id": "low_request_rate_apache", + "config": { + "job_id": "low_request_rate_apache", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "query": { + "bool": { + "filter": [ + { + "term": { + "data_stream.dataset": "apache.access" + } + } + ] + } + }, + "aggregations": { + "buckets": { + "date_histogram": { + "field": "@timestamp", + "fixed_interval": "15m", + "offset": 0, + "order": { + "_key": "asc" + }, + "keyed": false, + "min_doc_count": 0 + }, + "aggregations": { + "@timestamp": { + "max": { + "field": "@timestamp" + } + } + } + } + } + } + } + ] + }, + "id": "apache-Logs-ml", + "migrationVersion": { + "search": "7.9.3" + }, + "references": [], + "type": "ml-module" +} \ No newline at end of file diff --git a/test/packages/apache/kibana/search/apache-HTTPD.json b/test/packages/apache/kibana/search/apache-HTTPD.json index 73b560cd5a..5afdb95ba7 100644 --- a/test/packages/apache/kibana/search/apache-HTTPD.json +++ b/test/packages/apache/kibana/search/apache-HTTPD.json @@ -39,7 +39,7 @@ }, "id": "apache-HTTPD", "migrationVersion": { - "search": "7.4.0" + "search": "7.9.3" }, "references": [ { diff --git a/test/packages/apache/kibana/search/apache-access-logs.json b/test/packages/apache/kibana/search/apache-access-logs.json index 8571f8fbe1..5d2d2e31b5 100644 --- a/test/packages/apache/kibana/search/apache-access-logs.json +++ b/test/packages/apache/kibana/search/apache-access-logs.json @@ -42,7 +42,7 @@ }, "id": "apache-access-logs", "migrationVersion": { - "search": "7.4.0" + "search": "7.9.3" }, "references": [ { diff --git a/test/packages/apache/kibana/search/apache-errors-log.json b/test/packages/apache/kibana/search/apache-errors-log.json index ff2bf239b2..ff5742a373 100644 --- a/test/packages/apache/kibana/search/apache-errors-log.json +++ b/test/packages/apache/kibana/search/apache-errors-log.json @@ -42,7 +42,7 @@ }, "id": "apache-errors-log", "migrationVersion": { - "search": "7.4.0" + "search": "7.9.3" }, "references": [ { diff --git a/test/packages/apache/kibana/visualization/apache-22057f20-3a12-11eb-8946-296aab7b13db.json b/test/packages/apache/kibana/visualization/apache-22057f20-3a12-11eb-8946-296aab7b13db.json new file mode 100644 index 0000000000..c7d5744883 --- /dev/null +++ b/test/packages/apache/kibana/visualization/apache-22057f20-3a12-11eb-8946-296aab7b13db.json @@ -0,0 +1,78 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Uptime [Metrics Apache]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "background_color_rules": [ + { + "id": "c3c89690-3a11-11eb-8a27-5ff1727df0e0" + } + ], + "bar_color_rules": [ + { + "id": "c2fc9400-3a11-11eb-8a27-5ff1727df0e0" + } + ], + "default_index_pattern": "metrics-*", + "default_timefield": "@timestamp", + "id": "61ca57f0-469d-11e7-af02-69e470af7417", + "index_pattern": "", + "interval": "", + "isModelInvalid": false, + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "formatter": "s,humanize,", + "id": "61ca57f1-469d-11e7-af02-69e470af7417", + "label": "Uptime", + "line_width": 1, + "metrics": [ + { + "field": "apache.status.uptime.uptime", + "id": "61ca57f2-469d-11e7-af02-69e470af7417", + "type": "max" + } + ], + "point_size": 1, + "separate_axis": 0, + "split_color_mode": "kibana", + "split_mode": "everything", + "stacked": "none" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "", + "tooltip_mode": "show_all", + "type": "metric" + }, + "title": "Uptime [Metrics Apache]", + "type": "metrics" + } + }, + "id": "apache-22057f20-3a12-11eb-8946-296aab7b13db", + "migrationVersion": { + "visualization": "7.9.3" + }, + "references": [], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/apache/kibana/visualization/apache-320cd980-3a36-11eb-8946-296aab7b13db.json b/test/packages/apache/kibana/visualization/apache-320cd980-3a36-11eb-8946-296aab7b13db.json new file mode 100644 index 0000000000..66fe71c6f4 --- /dev/null +++ b/test/packages/apache/kibana/visualization/apache-320cd980-3a36-11eb-8946-296aab7b13db.json @@ -0,0 +1,152 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "savedSearchRefName": "search_0", + "title": "Requests per sec [Metrics Apache]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "Requests per sec", + "field": "apache.status.requests_per_sec" + }, + "schema": "metric", + "type": "avg" + }, + { + "enabled": true, + "id": "2", + "params": { + "drop_partials": false, + "extended_bounds": {}, + "field": "@timestamp", + "interval": "auto", + "min_doc_count": 1, + "scaleMetricValues": false, + "timeRange": { + "from": "now-15m", + "to": "now" + }, + "useNormalizedEsInterval": true + }, + "schema": "segment", + "type": "date_histogram" + } + ], + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "categoryAxes": [ + { + "id": "CategoryAxis-1", + "labels": { + "filter": true, + "show": true, + "truncate": 100 + }, + "position": "bottom", + "scale": { + "type": "linear" + }, + "show": true, + "style": {}, + "title": {}, + "type": "category" + } + ], + "defaultYExtents": false, + "drawLinesBetweenPoints": true, + "grid": { + "categoryLines": false + }, + "interpolate": "linear", + "labels": {}, + "legendPosition": "right", + "radiusRatio": 9, + "row": true, + "scale": "linear", + "seriesParams": [ + { + "data": { + "id": "1", + "label": "Requests per sec" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + } + ], + "setYExtents": false, + "shareYAxis": true, + "showCircles": true, + "smoothLines": false, + "thresholdLine": { + "color": "#E7664C", + "show": false, + "style": "full", + "value": 10, + "width": 1 + }, + "times": [], + "type": "line", + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "LeftAxis-1", + "position": "left", + "scale": { + "mode": "normal", + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Requests per sec" + }, + "type": "value" + } + ], + "yAxis": {} + }, + "title": "Requests per sec [Metrics Apache]", + "type": "line" + } + }, + "id": "apache-320cd980-3a36-11eb-8946-296aab7b13db", + "migrationVersion": { + "visualization": "7.9.3" + }, + "references": [ + { + "id": "apache-HTTPD", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/apache/kibana/visualization/apache-47820ce0-3a1d-11eb-8946-296aab7b13db.json b/test/packages/apache/kibana/visualization/apache-47820ce0-3a1d-11eb-8946-296aab7b13db.json new file mode 100644 index 0000000000..232a587a63 --- /dev/null +++ b/test/packages/apache/kibana/visualization/apache-47820ce0-3a1d-11eb-8946-296aab7b13db.json @@ -0,0 +1,89 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Total accesses [Metrics Apache]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "background_color_rules": [ + { + "id": "5414c4a0-3a1a-11eb-8b9a-851db9ca6ca8" + } + ], + "bar_color_rules": [ + { + "id": "c532ace0-3a1c-11eb-8b9a-851db9ca6ca8" + } + ], + "default_index_pattern": "metrics-*", + "default_timefield": "@timestamp", + "gauge_color_rules": [ + { + "id": "586a5890-3a19-11eb-8b9a-851db9ca6ca8" + } + ], + "gauge_inner_width": 10, + "gauge_style": "half", + "gauge_width": 10, + "id": "61ca57f0-469d-11e7-af02-69e470af7417", + "index_pattern": "", + "interval": "", + "isModelInvalid": false, + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "filter": { + "language": "kuery", + "query": "" + }, + "formatter": "0a", + "id": "6ccbc140-3a1c-11eb-8b9a-851db9ca6ca8", + "label": "Total accesses", + "line_width": 1, + "metrics": [ + { + "field": "apache.status.total_accesses", + "id": "6ccbc141-3a1c-11eb-8b9a-851db9ca6ca8", + "type": "max" + } + ], + "point_size": 1, + "separate_axis": 0, + "split_mode": "everything", + "stacked": "none" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "", + "tooltip_mode": "show_all", + "type": "metric" + }, + "title": "Total accesses [Metrics Apache]", + "type": "metrics" + } + }, + "id": "apache-47820ce0-3a1d-11eb-8946-296aab7b13db", + "migrationVersion": { + "visualization": "7.9.3" + }, + "references": [], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/apache/kibana/visualization/apache-7724cf20-3a39-11eb-8946-296aab7b13db.json b/test/packages/apache/kibana/visualization/apache-7724cf20-3a39-11eb-8946-296aab7b13db.json new file mode 100644 index 0000000000..6c7e554b74 --- /dev/null +++ b/test/packages/apache/kibana/visualization/apache-7724cf20-3a39-11eb-8946-296aab7b13db.json @@ -0,0 +1,189 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Connections [Metrics Apache]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "Writing", + "field": "apache.status.connections.async.writing" + }, + "schema": "metric", + "type": "max" + }, + { + "enabled": true, + "id": "2", + "params": { + "drop_partials": false, + "extended_bounds": {}, + "field": "@timestamp", + "interval": "30s", + "min_doc_count": 1, + "scaleMetricValues": false, + "timeRange": { + "from": "now-15m", + "to": "now" + }, + "useNormalizedEsInterval": true + }, + "schema": "segment", + "type": "date_histogram" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "Keep alive", + "field": "apache.status.connections.async.keep_alive" + }, + "schema": "metric", + "type": "max" + }, + { + "enabled": true, + "id": "4", + "params": { + "customLabel": "Closing", + "field": "apache.status.connections.async.closing" + }, + "schema": "metric", + "type": "max" + } + ], + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "categoryAxes": [ + { + "id": "CategoryAxis-1", + "labels": { + "filter": true, + "show": true, + "truncate": 100 + }, + "position": "bottom", + "scale": { + "type": "linear" + }, + "show": true, + "style": {}, + "title": {}, + "type": "category" + } + ], + "grid": { + "categoryLines": false + }, + "labels": {}, + "legendPosition": "right", + "seriesParams": [ + { + "data": { + "id": "1", + "label": "Writing" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + }, + { + "data": { + "id": "3", + "label": "Keep alive" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + }, + { + "data": { + "id": "4", + "label": "Closing" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + } + ], + "thresholdLine": { + "color": "#E7664C", + "show": false, + "style": "full", + "value": 10, + "width": 1 + }, + "times": [], + "type": "line", + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "LeftAxis-1", + "position": "left", + "scale": { + "mode": "normal", + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Connections" + }, + "type": "value" + } + ] + }, + "title": "Connections [Metrics Apache]", + "type": "line" + } + }, + "id": "apache-7724cf20-3a39-11eb-8946-296aab7b13db", + "migrationVersion": { + "visualization": "7.9.3" + }, + "references": [ + { + "id": "metrics-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/apache/kibana/visualization/apache-7d68f730-3a39-11eb-8946-296aab7b13db.json b/test/packages/apache/kibana/visualization/apache-7d68f730-3a39-11eb-8946-296aab7b13db.json new file mode 100644 index 0000000000..0f925bc270 --- /dev/null +++ b/test/packages/apache/kibana/visualization/apache-7d68f730-3a39-11eb-8946-296aab7b13db.json @@ -0,0 +1,141 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Total connections [Metrics Apache]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "2", + "params": { + "drop_partials": false, + "extended_bounds": {}, + "field": "@timestamp", + "interval": "30s", + "min_doc_count": 1, + "scaleMetricValues": false, + "timeRange": { + "from": "now-15m", + "to": "now" + }, + "useNormalizedEsInterval": true + }, + "schema": "segment", + "type": "date_histogram" + }, + { + "enabled": true, + "id": "4", + "params": { + "customLabel": "Total", + "field": "apache.status.connections.total" + }, + "schema": "metric", + "type": "max" + } + ], + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "categoryAxes": [ + { + "id": "CategoryAxis-1", + "labels": { + "filter": true, + "show": true, + "truncate": 100 + }, + "position": "bottom", + "scale": { + "type": "linear" + }, + "show": true, + "style": {}, + "title": {}, + "type": "category" + } + ], + "grid": { + "categoryLines": false + }, + "labels": {}, + "legendPosition": "right", + "seriesParams": [ + { + "data": { + "id": "4", + "label": "Total" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + } + ], + "thresholdLine": { + "color": "#E7664C", + "show": false, + "style": "full", + "value": 10, + "width": 1 + }, + "times": [], + "type": "line", + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "LeftAxis-1", + "position": "left", + "scale": { + "mode": "normal", + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Connections" + }, + "type": "value" + } + ] + }, + "title": "Total connections [Metrics Apache]", + "type": "line" + } + }, + "id": "apache-7d68f730-3a39-11eb-8946-296aab7b13db", + "migrationVersion": { + "visualization": "7.9.3" + }, + "references": [ + { + "id": "metrics-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/apache/kibana/visualization/apache-805d7bb0-3a10-11eb-8946-296aab7b13db.json b/test/packages/apache/kibana/visualization/apache-805d7bb0-3a10-11eb-8946-296aab7b13db.json new file mode 100644 index 0000000000..c663484193 --- /dev/null +++ b/test/packages/apache/kibana/visualization/apache-805d7bb0-3a10-11eb-8946-296aab7b13db.json @@ -0,0 +1,56 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Apache Hostname [Metrics Apache]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "controls": [ + { + "fieldName": "host.hostname", + "id": "1607512709833", + "indexPatternRefName": "control_0_index_pattern", + "label": "Hostname", + "options": { + "dynamicOptions": true, + "multiselect": false, + "order": "desc", + "size": 5, + "type": "terms" + }, + "parent": "", + "type": "list" + } + ], + "pinFilters": false, + "updateFiltersOnChange": true, + "useTimeFilter": false + }, + "title": "Apache Hostname [Metrics Apache]", + "type": "input_control_vis" + } + }, + "id": "apache-805d7bb0-3a10-11eb-8946-296aab7b13db", + "migrationVersion": { + "visualization": "7.9.3" + }, + "references": [ + { + "id": "metrics-*", + "name": "control_0_index_pattern", + "type": "index-pattern" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/apache/kibana/visualization/apache-99666080-3a20-11eb-8946-296aab7b13db.json b/test/packages/apache/kibana/visualization/apache-99666080-3a20-11eb-8946-296aab7b13db.json new file mode 100644 index 0000000000..fcf0f34c90 --- /dev/null +++ b/test/packages/apache/kibana/visualization/apache-99666080-3a20-11eb-8946-296aab7b13db.json @@ -0,0 +1,90 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Total egress [Metrics Apache]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "background_color_rules": [ + { + "id": "5414c4a0-3a1a-11eb-8b9a-851db9ca6ca8" + } + ], + "bar_color_rules": [ + { + "id": "c532ace0-3a1c-11eb-8b9a-851db9ca6ca8" + } + ], + "default_index_pattern": "metrics-*", + "default_timefield": "@timestamp", + "gauge_color_rules": [ + { + "id": "586a5890-3a19-11eb-8b9a-851db9ca6ca8" + } + ], + "gauge_inner_width": 10, + "gauge_style": "half", + "gauge_width": 10, + "id": "61ca57f0-469d-11e7-af02-69e470af7417", + "index_pattern": "", + "interval": "", + "isModelInvalid": false, + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "filter": { + "language": "kuery", + "query": "" + }, + "formatter": "bytes", + "id": "61ca57f1-469d-11e7-af02-69e470af7417", + "label": "Total egress", + "line_width": 1, + "metrics": [ + { + "field": "apache.status.total_bytes", + "id": "61ca57f2-469d-11e7-af02-69e470af7417", + "type": "max" + } + ], + "point_size": 1, + "separate_axis": 0, + "split_color_mode": "kibana", + "split_mode": "everything", + "stacked": "none" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "", + "tooltip_mode": "show_all", + "type": "metric" + }, + "title": "Total egress [Metrics Apache]", + "type": "metrics" + } + }, + "id": "apache-99666080-3a20-11eb-8946-296aab7b13db", + "migrationVersion": { + "visualization": "7.9.3" + }, + "references": [], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/apache/kibana/visualization/apache-HTTPD-CPU.json b/test/packages/apache/kibana/visualization/apache-HTTPD-CPU.json index 5fce62e7a5..dc0b8c8496 100644 --- a/test/packages/apache/kibana/visualization/apache-HTTPD-CPU.json +++ b/test/packages/apache/kibana/visualization/apache-HTTPD-CPU.json @@ -3,7 +3,11 @@ "description": "", "kibanaSavedObjectMeta": { "searchSourceJSON": { - "filter": [] + "filter": [], + "query": { + "language": "kuery", + "query": "" + } } }, "savedSearchRefName": "search_0", @@ -13,6 +17,7 @@ "visState": { "aggs": [ { + "enabled": true, "id": "1", "params": { "customLabel": "CPU load", @@ -22,28 +27,42 @@ "type": "avg" }, { + "enabled": true, "id": "2", "params": { + "drop_partials": false, "extended_bounds": {}, "field": "@timestamp", "interval": "auto", - "min_doc_count": 1 + "min_doc_count": 1, + "scaleMetricValues": false, + "timeRange": { + "from": "now-15m", + "to": "now" + }, + "useNormalizedEsInterval": true }, "schema": "segment", "type": "date_histogram" }, { + "enabled": true, "id": "3", "params": { - "field": "apache.status.hostname", + "field": "host.hostname", + "missingBucket": false, + "missingBucketLabel": "Missing", "order": "desc", "orderBy": "1", + "otherBucket": false, + "otherBucketLabel": "Other", "size": 5 }, "schema": "split", "type": "terms" }, { + "enabled": true, "id": "4", "params": { "customLabel": "CPU user", @@ -53,6 +72,7 @@ "type": "avg" }, { + "enabled": true, "id": "5", "params": { "customLabel": "CPU system", @@ -62,6 +82,7 @@ "type": "avg" }, { + "enabled": true, "id": "6", "params": { "customLabel": "CPU children user", @@ -71,6 +92,7 @@ "type": "avg" }, { + "enabled": true, "id": "7", "params": { "customLabel": "CPU children system", @@ -80,31 +102,156 @@ "type": "avg" } ], - "listeners": {}, "params": { "addLegend": true, "addTimeMarker": false, "addTooltip": true, + "categoryAxes": [ + { + "id": "CategoryAxis-1", + "labels": { + "filter": true, + "show": true, + "truncate": 100 + }, + "position": "bottom", + "scale": { + "type": "linear" + }, + "show": true, + "style": {}, + "title": {}, + "type": "category" + } + ], "defaultYExtents": false, "drawLinesBetweenPoints": true, + "grid": { + "categoryLines": false + }, "interpolate": "linear", + "labels": {}, + "legendPosition": "right", "radiusRatio": 9, "row": true, "scale": "linear", + "seriesParams": [ + { + "data": { + "id": "1", + "label": "CPU load" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + }, + { + "data": { + "id": "4", + "label": "CPU user" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + }, + { + "data": { + "id": "5", + "label": "CPU system" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + }, + { + "data": { + "id": "6", + "label": "CPU children user" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + }, + { + "data": { + "id": "7", + "label": "CPU children system" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + } + ], "setYExtents": false, "shareYAxis": true, "showCircles": true, "smoothLines": false, + "thresholdLine": { + "color": "#E7664C", + "show": false, + "style": "full", + "value": 10, + "width": 1 + }, "times": [], + "type": "line", + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "LeftAxis-1", + "position": "left", + "scale": { + "mode": "normal", + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Count" + }, + "type": "value" + } + ], "yAxis": {} }, - "title": "Apache HTTPD - CPU", + "title": "CPU usage [Metrics Apache]", "type": "line" } }, "id": "apache-HTTPD-CPU", "migrationVersion": { - "visualization": "7.8.0" + "visualization": "7.9.3" }, "references": [ { diff --git a/test/packages/apache/kibana/visualization/apache-HTTPD-Hostname-list.json b/test/packages/apache/kibana/visualization/apache-HTTPD-Hostname-list.json deleted file mode 100644 index ea5454985c..0000000000 --- a/test/packages/apache/kibana/visualization/apache-HTTPD-Hostname-list.json +++ /dev/null @@ -1,71 +0,0 @@ -{ - "attributes": { - "description": "", - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [] - } - }, - "savedSearchRefName": "search_0", - "title": "Hostname list [Metrics Apache]", - "uiStateJSON": { - "vis": { - "params": { - "sort": { - "columnIndex": null, - "direction": null - } - } - } - }, - "version": 1, - "visState": { - "aggs": [ - { - "id": "1", - "params": { - "customLabel": "Events count" - }, - "schema": "metric", - "type": "count" - }, - { - "id": "2", - "params": { - "customLabel": "Apache HTTD Hostname", - "field": "apache.status.hostname", - "order": "desc", - "orderBy": "1", - "size": 5 - }, - "schema": "bucket", - "type": "terms" - } - ], - "listeners": {}, - "params": { - "perPage": 10, - "showMeticsAtAllLevels": false, - "showPartialRows": false, - "sort": { - "columnIndex": null, - "direction": null - } - }, - "title": "Apache HTTPD - Hostname list", - "type": "table" - } - }, - "id": "apache-HTTPD-Hostname-list", - "migrationVersion": { - "visualization": "7.8.0" - }, - "references": [ - { - "id": "apache-HTTPD", - "name": "search_0", - "type": "search" - } - ], - "type": "visualization" -} \ No newline at end of file diff --git a/test/packages/apache/kibana/visualization/apache-HTTPD-Load1-slash-5-slash-15.json b/test/packages/apache/kibana/visualization/apache-HTTPD-Load1-slash-5-slash-15.json index 488052c19d..8d0ba87a59 100644 --- a/test/packages/apache/kibana/visualization/apache-HTTPD-Load1-slash-5-slash-15.json +++ b/test/packages/apache/kibana/visualization/apache-HTTPD-Load1-slash-5-slash-15.json @@ -3,7 +3,11 @@ "description": "", "kibanaSavedObjectMeta": { "searchSourceJSON": { - "filter": [] + "filter": [], + "query": { + "language": "kuery", + "query": "" + } } }, "savedSearchRefName": "search_0", @@ -13,6 +17,7 @@ "visState": { "aggs": [ { + "enabled": true, "id": "1", "params": { "customLabel": "Load 5", @@ -22,17 +27,26 @@ "type": "avg" }, { + "enabled": true, "id": "2", "params": { + "drop_partials": false, "extended_bounds": {}, "field": "@timestamp", "interval": "auto", - "min_doc_count": 1 + "min_doc_count": 1, + "scaleMetricValues": false, + "timeRange": { + "from": "now-15m", + "to": "now" + }, + "useNormalizedEsInterval": true }, "schema": "segment", "type": "date_histogram" }, { + "enabled": true, "id": "3", "params": { "customLabel": "Load 1", @@ -42,6 +56,7 @@ "type": "avg" }, { + "enabled": true, "id": "4", "params": { "customLabel": "Load 15", @@ -51,43 +66,145 @@ "type": "avg" }, { + "enabled": true, "id": "5", "params": { "customLabel": "Hostname", - "field": "apache.status.hostname", + "field": "host.hostname", + "missingBucket": false, + "missingBucketLabel": "Missing", "order": "desc", "orderBy": "1", + "otherBucket": false, + "otherBucketLabel": "Other", "size": 5 }, "schema": "split", "type": "terms" } ], - "listeners": {}, "params": { "addLegend": true, "addTimeMarker": false, "addTooltip": true, + "categoryAxes": [ + { + "id": "CategoryAxis-1", + "labels": { + "filter": true, + "show": true, + "truncate": 100 + }, + "position": "bottom", + "scale": { + "type": "linear" + }, + "show": true, + "style": {}, + "title": {}, + "type": "category" + } + ], "defaultYExtents": false, "drawLinesBetweenPoints": true, + "grid": { + "categoryLines": false + }, "interpolate": "linear", + "labels": {}, + "legendPosition": "right", "radiusRatio": 9, "row": true, "scale": "linear", + "seriesParams": [ + { + "data": { + "id": "1", + "label": "Load 5" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + }, + { + "data": { + "id": "3", + "label": "Load 1" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + }, + { + "data": { + "id": "4", + "label": "Load 15" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + } + ], "setYExtents": false, "shareYAxis": true, "showCircles": true, "smoothLines": false, + "thresholdLine": { + "color": "#E7664C", + "show": false, + "style": "full", + "value": 10, + "width": 1 + }, "times": [], + "type": "line", + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "LeftAxis-1", + "position": "left", + "scale": { + "mode": "normal", + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Count" + }, + "type": "value" + } + ], "yAxis": {} }, - "title": "Apache HTTPD - Load1/5/15", + "title": "Load1/5/15 [Metrics Apache]", "type": "line" } }, "id": "apache-HTTPD-Load1-slash-5-slash-15", "migrationVersion": { - "visualization": "7.8.0" + "visualization": "7.9.3" }, "references": [ { diff --git a/test/packages/apache/kibana/visualization/apache-HTTPD-Scoreboard.json b/test/packages/apache/kibana/visualization/apache-HTTPD-Scoreboard.json index 7af844ff47..1167e1092a 100644 --- a/test/packages/apache/kibana/visualization/apache-HTTPD-Scoreboard.json +++ b/test/packages/apache/kibana/visualization/apache-HTTPD-Scoreboard.json @@ -3,7 +3,11 @@ "description": "", "kibanaSavedObjectMeta": { "searchSourceJSON": { - "filter": [] + "filter": [], + "query": { + "language": "kuery", + "query": "" + } } }, "savedSearchRefName": "search_0", @@ -13,6 +17,7 @@ "visState": { "aggs": [ { + "enabled": true, "id": "1", "params": { "customLabel": "Closing connection", @@ -22,29 +27,43 @@ "type": "avg" }, { + "enabled": true, "id": "2", "params": { + "drop_partials": false, "extended_bounds": {}, "field": "@timestamp", "interval": "auto", - "min_doc_count": 1 + "min_doc_count": 1, + "scaleMetricValues": false, + "timeRange": { + "from": "now-15m", + "to": "now" + }, + "useNormalizedEsInterval": true }, "schema": "segment", "type": "date_histogram" }, { + "enabled": true, "id": "3", "params": { "customLabel": "Hostname", - "field": "apache.status.hostname", + "field": "host.hostname", + "missingBucket": false, + "missingBucketLabel": "Missing", "order": "desc", "orderBy": "1", + "otherBucket": false, + "otherBucketLabel": "Other", "size": 5 }, "schema": "split", "type": "terms" }, { + "enabled": true, "id": "4", "params": { "customLabel": "DNS lookup", @@ -54,6 +73,7 @@ "type": "avg" }, { + "enabled": true, "id": "5", "params": { "customLabel": "Gracefully finishing", @@ -63,6 +83,7 @@ "type": "avg" }, { + "enabled": true, "id": "6", "params": { "customLabel": "Idle cleanup", @@ -72,6 +93,7 @@ "type": "avg" }, { + "enabled": true, "id": "7", "params": { "customLabel": "Keepalive", @@ -81,6 +103,7 @@ "type": "avg" }, { + "enabled": true, "id": "8", "params": { "customLabel": "Logging", @@ -90,6 +113,7 @@ "type": "avg" }, { + "enabled": true, "id": "9", "params": { "customLabel": "Open slot", @@ -99,6 +123,7 @@ "type": "avg" }, { + "enabled": true, "id": "10", "params": { "customLabel": "Reading request", @@ -108,6 +133,7 @@ "type": "avg" }, { + "enabled": true, "id": "11", "params": { "customLabel": "Sending reply", @@ -117,6 +143,7 @@ "type": "avg" }, { + "enabled": true, "id": "12", "params": { "customLabel": "Starting up", @@ -126,6 +153,7 @@ "type": "avg" }, { + "enabled": true, "id": "13", "params": { "customLabel": "Total", @@ -135,6 +163,7 @@ "type": "avg" }, { + "enabled": true, "id": "14", "params": { "customLabel": "Waiting for connection", @@ -144,31 +173,254 @@ "type": "avg" } ], - "listeners": {}, "params": { "addLegend": true, "addTimeMarker": false, "addTooltip": true, + "categoryAxes": [ + { + "id": "CategoryAxis-1", + "labels": { + "filter": true, + "show": true, + "truncate": 100 + }, + "position": "bottom", + "scale": { + "type": "linear" + }, + "show": true, + "style": {}, + "title": {}, + "type": "category" + } + ], "defaultYExtents": false, "drawLinesBetweenPoints": true, + "grid": { + "categoryLines": false + }, "interpolate": "linear", + "labels": {}, + "legendPosition": "right", "radiusRatio": 9, "row": true, "scale": "linear", + "seriesParams": [ + { + "data": { + "id": "1", + "label": "Closing connection" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + }, + { + "data": { + "id": "4", + "label": "DNS lookup" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + }, + { + "data": { + "id": "5", + "label": "Gracefully finishing" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + }, + { + "data": { + "id": "6", + "label": "Idle cleanup" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + }, + { + "data": { + "id": "7", + "label": "Keepalive" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + }, + { + "data": { + "id": "8", + "label": "Logging" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + }, + { + "data": { + "id": "9", + "label": "Open slot" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + }, + { + "data": { + "id": "10", + "label": "Reading request" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + }, + { + "data": { + "id": "11", + "label": "Sending reply" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + }, + { + "data": { + "id": "12", + "label": "Starting up" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + }, + { + "data": { + "id": "13", + "label": "Total" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + }, + { + "data": { + "id": "14", + "label": "Waiting for connection" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + } + ], "setYExtents": false, "shareYAxis": true, "showCircles": true, "smoothLines": false, + "thresholdLine": { + "color": "#E7664C", + "show": false, + "style": "full", + "value": 10, + "width": 1 + }, "times": [], + "type": "line", + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "LeftAxis-1", + "position": "left", + "scale": { + "mode": "normal", + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Count" + }, + "type": "value" + } + ], "yAxis": {} }, - "title": "Apache HTTPD - Scoreboard", + "title": "Scoreboard [Metrics Apache]", "type": "line" } }, "id": "apache-HTTPD-Scoreboard", "migrationVersion": { - "visualization": "7.8.0" + "visualization": "7.9.3" }, "references": [ { diff --git a/test/packages/apache/kibana/visualization/apache-HTTPD-Total-accesses-and-kbytes.json b/test/packages/apache/kibana/visualization/apache-HTTPD-Total-accesses-and-kbytes.json deleted file mode 100644 index b4adcccb0d..0000000000 --- a/test/packages/apache/kibana/visualization/apache-HTTPD-Total-accesses-and-kbytes.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "attributes": { - "description": "", - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [] - } - }, - "savedSearchRefName": "search_0", - "title": "Total accesses and kbytes [Metrics Apache]", - "uiStateJSON": {}, - "version": 1, - "visState": { - "aggs": [ - { - "id": "1", - "params": { - "customLabel": "Total kbytes", - "field": "apache.status.total_bytes" - }, - "schema": "metric", - "type": "max" - }, - { - "id": "2", - "params": { - "customLabel": "Total accesses", - "field": "apache.status.total_accesses" - }, - "schema": "metric", - "type": "max" - } - ], - "listeners": {}, - "params": { - "fontSize": 60, - "handleNoResults": true - }, - "title": "Apache HTTPD - Total accesses and kbytes", - "type": "metric" - } - }, - "id": "apache-HTTPD-Total-accesses-and-kbytes", - "migrationVersion": { - "visualization": "7.8.0" - }, - "references": [ - { - "id": "apache-HTTPD", - "name": "search_0", - "type": "search" - } - ], - "type": "visualization" -} \ No newline at end of file diff --git a/test/packages/apache/kibana/visualization/apache-HTTPD-Uptime.json b/test/packages/apache/kibana/visualization/apache-HTTPD-Uptime.json deleted file mode 100644 index f5a9d3704d..0000000000 --- a/test/packages/apache/kibana/visualization/apache-HTTPD-Uptime.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "attributes": { - "description": "", - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [] - } - }, - "savedSearchRefName": "search_0", - "title": "Uptime [Metrics Apache]", - "uiStateJSON": {}, - "version": 1, - "visState": { - "aggs": [ - { - "id": "1", - "params": { - "customLabel": "Uptime", - "field": "apache.status.uptime.uptime" - }, - "schema": "metric", - "type": "max" - }, - { - "id": "2", - "params": { - "customLabel": "Server uptime", - "field": "apache.status.uptime.server_uptime" - }, - "schema": "metric", - "type": "max" - } - ], - "listeners": {}, - "params": { - "fontSize": 60, - "handleNoResults": true - }, - "title": "Apache HTTPD - Uptime", - "type": "metric" - } - }, - "id": "apache-HTTPD-Uptime", - "migrationVersion": { - "visualization": "7.8.0" - }, - "references": [ - { - "id": "apache-HTTPD", - "name": "search_0", - "type": "search" - } - ], - "type": "visualization" -} \ No newline at end of file diff --git a/test/packages/apache/kibana/visualization/apache-HTTPD-Workers.json b/test/packages/apache/kibana/visualization/apache-HTTPD-Workers.json deleted file mode 100644 index 6f54ccaadf..0000000000 --- a/test/packages/apache/kibana/visualization/apache-HTTPD-Workers.json +++ /dev/null @@ -1,91 +0,0 @@ -{ - "attributes": { - "description": "", - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [] - } - }, - "savedSearchRefName": "search_0", - "title": "Workers [Metrics Apache]", - "uiStateJSON": {}, - "version": 1, - "visState": { - "aggs": [ - { - "id": "1", - "params": { - "customLabel": "Busy workers", - "field": "apache.status.workers.busy" - }, - "schema": "metric", - "type": "avg" - }, - { - "id": "2", - "params": { - "extended_bounds": {}, - "field": "@timestamp", - "interval": "auto", - "min_doc_count": 1 - }, - "schema": "segment", - "type": "date_histogram" - }, - { - "id": "3", - "params": { - "customLabel": "Hostname", - "field": "apache.status.hostname", - "order": "desc", - "orderBy": "1", - "size": 5 - }, - "schema": "split", - "type": "terms" - }, - { - "id": "4", - "params": { - "customLabel": "Idle workers", - "field": "apache.status.workers.idle" - }, - "schema": "metric", - "type": "avg" - } - ], - "listeners": {}, - "params": { - "addLegend": true, - "addTimeMarker": false, - "addTooltip": true, - "defaultYExtents": false, - "drawLinesBetweenPoints": true, - "interpolate": "linear", - "radiusRatio": 9, - "row": true, - "scale": "linear", - "setYExtents": false, - "shareYAxis": true, - "showCircles": true, - "smoothLines": false, - "times": [], - "yAxis": {} - }, - "title": "Apache HTTPD - Workers", - "type": "line" - } - }, - "id": "apache-HTTPD-Workers", - "migrationVersion": { - "visualization": "7.8.0" - }, - "references": [ - { - "id": "apache-HTTPD", - "name": "search_0", - "type": "search" - } - ], - "type": "visualization" -} \ No newline at end of file diff --git a/test/packages/apache/kibana/visualization/apache-a45311f0-3a34-11eb-8946-296aab7b13db.json b/test/packages/apache/kibana/visualization/apache-a45311f0-3a34-11eb-8946-296aab7b13db.json new file mode 100644 index 0000000000..ec859d0cc0 --- /dev/null +++ b/test/packages/apache/kibana/visualization/apache-a45311f0-3a34-11eb-8946-296aab7b13db.json @@ -0,0 +1,152 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "savedSearchRefName": "search_0", + "title": "Bytes per sec [Metrics Apache]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "Bytes per sec", + "field": "apache.status.bytes_per_sec" + }, + "schema": "metric", + "type": "avg" + }, + { + "enabled": true, + "id": "2", + "params": { + "drop_partials": false, + "extended_bounds": {}, + "field": "@timestamp", + "interval": "auto", + "min_doc_count": 1, + "scaleMetricValues": false, + "timeRange": { + "from": "now-15m", + "to": "now" + }, + "useNormalizedEsInterval": true + }, + "schema": "segment", + "type": "date_histogram" + } + ], + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "categoryAxes": [ + { + "id": "CategoryAxis-1", + "labels": { + "filter": true, + "show": true, + "truncate": 100 + }, + "position": "bottom", + "scale": { + "type": "linear" + }, + "show": true, + "style": {}, + "title": {}, + "type": "category" + } + ], + "defaultYExtents": false, + "drawLinesBetweenPoints": true, + "grid": { + "categoryLines": false + }, + "interpolate": "linear", + "labels": {}, + "legendPosition": "right", + "radiusRatio": 9, + "row": true, + "scale": "linear", + "seriesParams": [ + { + "data": { + "id": "1", + "label": "Bytes per sec" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + } + ], + "setYExtents": false, + "shareYAxis": true, + "showCircles": true, + "smoothLines": false, + "thresholdLine": { + "color": "#E7664C", + "show": false, + "style": "full", + "value": 10, + "width": 1 + }, + "times": [], + "type": "line", + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "LeftAxis-1", + "position": "left", + "scale": { + "mode": "normal", + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Bytes per sec" + }, + "type": "value" + } + ], + "yAxis": {} + }, + "title": "Bytes per sec [Metrics Apache]", + "type": "line" + } + }, + "id": "apache-a45311f0-3a34-11eb-8946-296aab7b13db", + "migrationVersion": { + "visualization": "7.9.3" + }, + "references": [ + { + "id": "apache-HTTPD", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/apache/kibana/visualization/apache-access-unique-IPs-map.json b/test/packages/apache/kibana/visualization/apache-access-unique-IPs-map.json index 1263059c4e..4504b84a68 100644 --- a/test/packages/apache/kibana/visualization/apache-access-unique-IPs-map.json +++ b/test/packages/apache/kibana/visualization/apache-access-unique-IPs-map.json @@ -72,7 +72,7 @@ }, "id": "apache-access-unique-IPs-map", "migrationVersion": { - "visualization": "7.8.0" + "visualization": "7.9.3" }, "references": [ { diff --git a/test/packages/apache/kibana/visualization/apache-browsers.json b/test/packages/apache/kibana/visualization/apache-browsers.json index 6506bebf8b..5cc8e7bc52 100644 --- a/test/packages/apache/kibana/visualization/apache-browsers.json +++ b/test/packages/apache/kibana/visualization/apache-browsers.json @@ -60,7 +60,7 @@ }, "id": "apache-browsers", "migrationVersion": { - "visualization": "7.8.0" + "visualization": "7.9.3" }, "references": [ { diff --git a/test/packages/apache/kibana/visualization/apache-ed44f820-3a10-11eb-8946-296aab7b13db.json b/test/packages/apache/kibana/visualization/apache-ed44f820-3a10-11eb-8946-296aab7b13db.json new file mode 100644 index 0000000000..bc84a0c66d --- /dev/null +++ b/test/packages/apache/kibana/visualization/apache-ed44f820-3a10-11eb-8946-296aab7b13db.json @@ -0,0 +1,56 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Apache Hostname [Logs Apache]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "controls": [ + { + "fieldName": "host.hostname", + "id": "1607512709833", + "indexPatternRefName": "control_0_index_pattern", + "label": "Hostname", + "options": { + "dynamicOptions": true, + "multiselect": false, + "order": "desc", + "size": 5, + "type": "terms" + }, + "parent": "", + "type": "list" + } + ], + "pinFilters": false, + "updateFiltersOnChange": true, + "useTimeFilter": false + }, + "title": "Apache Hostname [Logs Apache]", + "type": "input_control_vis" + } + }, + "id": "apache-ed44f820-3a10-11eb-8946-296aab7b13db", + "migrationVersion": { + "visualization": "7.9.3" + }, + "references": [ + { + "id": "logs-*", + "name": "control_0_index_pattern", + "type": "index-pattern" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/apache/kibana/visualization/apache-error-logs-over-time.json b/test/packages/apache/kibana/visualization/apache-error-logs-over-time.json index 480a4a2750..bb97fac6a7 100644 --- a/test/packages/apache/kibana/visualization/apache-error-logs-over-time.json +++ b/test/packages/apache/kibana/visualization/apache-error-logs-over-time.json @@ -64,7 +64,7 @@ }, "id": "apache-error-logs-over-time", "migrationVersion": { - "visualization": "7.8.0" + "visualization": "7.9.3" }, "references": [ { diff --git a/test/packages/apache/kibana/visualization/apache-f4ffec70-3a36-11eb-8946-296aab7b13db.json b/test/packages/apache/kibana/visualization/apache-f4ffec70-3a36-11eb-8946-296aab7b13db.json new file mode 100644 index 0000000000..fad15583fb --- /dev/null +++ b/test/packages/apache/kibana/visualization/apache-f4ffec70-3a36-11eb-8946-296aab7b13db.json @@ -0,0 +1,180 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "savedSearchRefName": "search_0", + "title": "Workers [Metrics Apache]", + "uiStateJSON": { + "vis": { + "legendOpen": true + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "Busy workers", + "field": "apache.status.workers.busy" + }, + "schema": "metric", + "type": "avg" + }, + { + "enabled": true, + "id": "2", + "params": { + "drop_partials": false, + "extended_bounds": {}, + "field": "@timestamp", + "interval": "auto", + "min_doc_count": 1, + "scaleMetricValues": false, + "timeRange": { + "from": "now-15m", + "to": "now" + }, + "useNormalizedEsInterval": true + }, + "schema": "segment", + "type": "date_histogram" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "Idle workers", + "field": "apache.status.workers.idle" + }, + "schema": "metric", + "type": "avg" + } + ], + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "categoryAxes": [ + { + "id": "CategoryAxis-1", + "labels": { + "filter": true, + "show": true, + "truncate": 100 + }, + "position": "bottom", + "scale": { + "type": "linear" + }, + "show": true, + "style": {}, + "title": {}, + "type": "category" + } + ], + "defaultYExtents": false, + "drawLinesBetweenPoints": true, + "grid": { + "categoryLines": false + }, + "interpolate": "linear", + "labels": {}, + "legendPosition": "right", + "radiusRatio": 9, + "row": true, + "scale": "linear", + "seriesParams": [ + { + "data": { + "id": "1", + "label": "Busy workers" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + }, + { + "data": { + "id": "3", + "label": "Idle workers" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + } + ], + "setYExtents": false, + "shareYAxis": true, + "showCircles": true, + "smoothLines": false, + "thresholdLine": { + "color": "#E7664C", + "show": false, + "style": "full", + "value": 10, + "width": 1 + }, + "times": [], + "type": "line", + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "filter": false, + "rotate": 0, + "show": false, + "truncate": 100 + }, + "name": "LeftAxis-1", + "position": "left", + "scale": { + "mode": "normal", + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Workers" + }, + "type": "value" + } + ], + "yAxis": {} + }, + "title": "Workers [Metrics Apache]", + "type": "line" + } + }, + "id": "apache-f4ffec70-3a36-11eb-8946-296aab7b13db", + "migrationVersion": { + "visualization": "7.9.3" + }, + "references": [ + { + "id": "apache-HTTPD", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/apache/kibana/visualization/apache-operating-systems.json b/test/packages/apache/kibana/visualization/apache-operating-systems.json index 0d371fb525..196f983cd6 100644 --- a/test/packages/apache/kibana/visualization/apache-operating-systems.json +++ b/test/packages/apache/kibana/visualization/apache-operating-systems.json @@ -60,7 +60,7 @@ }, "id": "apache-operating-systems", "migrationVersion": { - "visualization": "7.8.0" + "visualization": "7.9.3" }, "references": [ { diff --git a/test/packages/apache/kibana/visualization/apache-response-codes-of-top-URLs.json b/test/packages/apache/kibana/visualization/apache-response-codes-of-top-URLs.json index 6a27d092a1..6b8eb93302 100644 --- a/test/packages/apache/kibana/visualization/apache-response-codes-of-top-URLs.json +++ b/test/packages/apache/kibana/visualization/apache-response-codes-of-top-URLs.json @@ -67,7 +67,7 @@ }, "id": "apache-response-codes-of-top-URLs", "migrationVersion": { - "visualization": "7.8.0" + "visualization": "7.9.3" }, "references": [ { diff --git a/test/packages/apache/kibana/visualization/apache-response-codes-over-time.json b/test/packages/apache/kibana/visualization/apache-response-codes-over-time.json index 2847a150fc..8f36ed2601 100644 --- a/test/packages/apache/kibana/visualization/apache-response-codes-over-time.json +++ b/test/packages/apache/kibana/visualization/apache-response-codes-over-time.json @@ -71,7 +71,7 @@ }, "id": "apache-response-codes-over-time", "migrationVersion": { - "visualization": "7.8.0" + "visualization": "7.9.3" }, "references": [ { diff --git a/test/packages/apache/manifest.yml b/test/packages/apache/manifest.yml index 47e87b9896..b04065d21c 100644 --- a/test/packages/apache/manifest.yml +++ b/test/packages/apache/manifest.yml @@ -1,23 +1,26 @@ format_version: 1.0.0 name: apache -title: Apache -version: 0.0.1 +title: Apache HTTP Server +# version is set to something very large to so this test package can +# be installed in the package registry regardless of the version of +# the actual apache package in the registry at any given time. +version: 999.999.999 license: basic -description: Apache Integration +description: Collect logs and metrics from Apache servers with Elastic Agent. type: integration categories: - web -release: experimental +release: ga conditions: - kibana.version: '^7.9.0' + kibana.version: "^7.14.0 || ^8.0.0" screenshots: - - src: /img/kibana-apache.png - title: Apache Integration - size: 1215x1199 + - src: /img/apache-metrics-overview.png + title: Apache metrics overview + size: 3360x3064 type: image/png - - src: /img/apache_httpd_server_status.png - title: Apache HTTPD Server Status - size: 1919x1079 + - src: /img/apache-logs-overview.png + title: Apache logs overview + size: 3342x1384 type: image/png icons: - src: /img/logo_apache.svg @@ -32,6 +35,65 @@ policy_templates: - type: logfile title: Collect logs from Apache instances description: Collecting Apache access and error logs + - type: httpjson + title: Collect logs from third-party REST API (experimental) + description: Collect logs from third-party REST API (experimental) + vars: + - name: url + type: text + title: URL of Splunk Enterprise Server + description: i.e. scheme://host:port, path is automatic + show_user: true + required: true + default: https://server.example.com:8089 + - name: username + type: text + title: Splunk REST API Username + show_user: true + required: false + - name: password + type: password + title: Splunk REST API Password + show_user: true + required: false + - name: token + type: password + title: Splunk Authorization Token + description: | + Bearer Token or Session Key, e.g. "Bearer eyJFd3e46..." + or "Splunk 192fd3e...". Cannot be used with username + and password. + show_user: true + required: false + - name: ssl + type: yaml + title: SSL Configuration + description: i.e. certificate_authorities, supported_protocols, verification_mode etc. + multi: false + required: false + show_user: false + default: | + #certificate_authorities: + # - | + # -----BEGIN CERTIFICATE----- + # MIIDCjCCAfKgAwIBAgITJ706Mu2wJlKckpIvkWxEHvEyijANBgkqhkiG9w0BAQsF + # ADAUMRIwEAYDVQQDDAlsb2NhbGhvc3QwIBcNMTkwNzIyMTkyOTA0WhgPMjExOTA2 + # MjgxOTI5MDRaMBQxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEB + # BQADggEPADCCAQoCggEBANce58Y/JykI58iyOXpxGfw0/gMvF0hUQAcUrSMxEO6n + # fZRA49b4OV4SwWmA3395uL2eB2NB8y8qdQ9muXUdPBWE4l9rMZ6gmfu90N5B5uEl + # 94NcfBfYOKi1fJQ9i7WKhTjlRkMCgBkWPkUokvBZFRt8RtF7zI77BSEorHGQCk9t + # /D7BS0GJyfVEhftbWcFEAG3VRcoMhF7kUzYwp+qESoriFRYLeDWv68ZOvG7eoWnP + # PsvZStEVEimjvK5NSESEQa9xWyJOmlOKXhkdymtcUd/nXnx6UTCFgnkgzSdTWV41 + # CI6B6aJ9svCTI2QuoIq2HxX/ix7OvW1huVmcyHVxyUECAwEAAaNTMFEwHQYDVR0O + # BBYEFPwN1OceFGm9v6ux8G+DZ3TUDYxqMB8GA1UdIwQYMBaAFPwN1OceFGm9v6ux + # 8G+DZ3TUDYxqMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAG5D + # 874A4YI7YUwOVsVAdbWtgp1d0zKcPRR+r2OdSbTAV5/gcS3jgBJ3i1BN34JuDVFw + # 3DeJSYT3nxy2Y56lLnxDeF8CUTUtVQx3CuGkRg1ouGAHpO/6OqOhwLLorEmxi7tA + # H2O8mtT0poX5AnOAhzVy7QW0D/k4WaoLyckM5hUa6RtvgvLxOwA0U+VGurCDoctu + # 8F4QOgTAWyh8EZIwaKCliFRSynDpv3JTUwtfZkxo6K6nce1RhCWFAsMvDZL8Dgc0 + # yvgJ38BRsFOtkRuAGSf6ZUwTO8JJRRIFnpUzXflAnGivK9M13D5GEQMmIl6U9Pvk + # sxSmbIUfc2SGJGCJD4I= + # -----END CERTIFICATE----- - type: apache/metrics title: Collect metrics from Apache instances description: Collecting Apache status metrics @@ -45,4 +107,4 @@ policy_templates: default: - http://127.0.0.1 owner: - github: elastic/integrations-services + github: elastic/integrations diff --git a/test/packages/aws/_dev/build/build.yml b/test/packages/aws/_dev/build/build.yml new file mode 100644 index 0000000000..08d85edcf9 --- /dev/null +++ b/test/packages/aws/_dev/build/build.yml @@ -0,0 +1,3 @@ +dependencies: + ecs: + reference: git@1.12 diff --git a/test/packages/aws/_dev/build/docs/README.md b/test/packages/aws/_dev/build/docs/README.md index e36b499695..c40242cdf6 100644 --- a/test/packages/aws/_dev/build/docs/README.md +++ b/test/packages/aws/_dev/build/docs/README.md @@ -15,6 +15,20 @@ AWS credentials are required for running AWS integration. * *endpoint*: URL of the entry point for an AWS web service. * *role_arn*: AWS IAM Role to assume. +#### Data stream specific configuration parameters +* *latency*: Some AWS services send monitoring metrics to CloudWatch with a +latency to process larger than Metricbeat collection period. This will cause +data points missing or none get collected by Metricbeat. In this case, please +specify a latency parameter so collection start time and end time will be +shifted by the given latency amount. +* *period*: How often the data stream is executed. +* *regions*: Specify which AWS regions to query metrics from. If the `regions` +is not set in the config, then by default, the integration will query metrics +from all available AWS regions. If `endpoint` is specified, `regions` becomes a +required config parameter. +* *tags_filter*: Tag key value pairs from aws resources. A tag is a label that +user assigns to an AWS resource. + ### Credential Types There are three types of AWS credentials can be used: access keys, temporary security credentials and IAM role ARN. @@ -61,10 +75,14 @@ temporary credentials. Please see for more details. ### Supported Formats -1. Use `access_key_id`, `secret_access_key` and/or `session_token` directly -2. Use `role_arn`: If `access_key_id` and `secret_access_key` are not given, -then the package will check for `role_arn`. `role_arn` is used to specify which - AWS IAM role to assume for generating temporary credentials. +1. Use access keys: Access keys include `access_key_id`, `secret_access_key` +and/or `session_token`. +2. Use `role_arn`: `role_arn` is used to specify which AWS IAM role to assume +for generating temporary credentials. If `role_arn` is given, the package will +check if access keys are given. If not, the package will check for credential +profile name. If neither is given, default credential profile will be used. +Please make sure credentials are given under either a credential profile or +access keys. 3. Use `credential_profile_name` and/or `shared_credential_file`: If `access_key_id`, `secret_access_key` and `role_arn` are all not given, then the package will check for `credential_profile_name`. If you use different diff --git a/test/packages/aws/_dev/build/docs/cloudtrail.md b/test/packages/aws/_dev/build/docs/cloudtrail.md index e06cbe105b..cbe7fb05c1 100644 --- a/test/packages/aws/_dev/build/docs/cloudtrail.md +++ b/test/packages/aws/_dev/build/docs/cloudtrail.md @@ -9,3 +9,5 @@ events for the account. If user creates a trail, it delivers those events as log Integrity is turned on, it only reads the CloudTrail logs. {{fields "cloudtrail"}} + +{{event "cloudtrail"}} diff --git a/test/packages/aws/_dev/build/docs/cloudwatch.md b/test/packages/aws/_dev/build/docs/cloudwatch.md index 50fe1a5a5f..a27d5f8c8b 100644 --- a/test/packages/aws/_dev/build/docs/cloudwatch.md +++ b/test/packages/aws/_dev/build/docs/cloudwatch.md @@ -9,6 +9,8 @@ setup already. {{fields "cloudwatch_logs"}} +{{event "cloudwatch_logs"}} + ## Metrics {{event "cloudwatch_metrics"}} diff --git a/test/packages/aws/_dev/build/docs/ec2.md b/test/packages/aws/_dev/build/docs/ec2.md index 9a9fe5e428..f0e6205075 100644 --- a/test/packages/aws/_dev/build/docs/ec2.md +++ b/test/packages/aws/_dev/build/docs/ec2.md @@ -9,6 +9,8 @@ and `process.name`. For logs from other services, please use `cloudwatch` datase {{fields "ec2_logs"}} +{{event "ec2_logs"}} + ## Metrics {{event "ec2_metrics"}} diff --git a/test/packages/aws/_dev/build/docs/elb.md b/test/packages/aws/_dev/build/docs/elb.md index 608ed45e66..db9413e39f 100644 --- a/test/packages/aws/_dev/build/docs/elb.md +++ b/test/packages/aws/_dev/build/docs/elb.md @@ -16,6 +16,8 @@ For network load balancer, please follow [enable access log for network load bal {{fields "elb_logs"}} +{{event "elb_logs"}} + ## Metrics {{event "elb_metrics"}} diff --git a/test/packages/aws/_dev/build/docs/s3.md b/test/packages/aws/_dev/build/docs/s3.md index cfa236aa13..ae3faed53b 100644 --- a/test/packages/aws/_dev/build/docs/s3.md +++ b/test/packages/aws/_dev/build/docs/s3.md @@ -12,6 +12,8 @@ for sending server access logs to S3 bucket. {{fields "s3access"}} +{{event "s3access"}} + ## Metrics ### s3_daily_storage diff --git a/test/packages/aws/_dev/build/docs/s3_storage_lens.md b/test/packages/aws/_dev/build/docs/s3_storage_lens.md new file mode 100644 index 0000000000..275ba37372 --- /dev/null +++ b/test/packages/aws/_dev/build/docs/s3_storage_lens.md @@ -0,0 +1,7 @@ +# s3 storage lens + +## Metrics + +{{event "s3_storage_lens"}} + +{{fields "s3_storage_lens"}} \ No newline at end of file diff --git a/test/packages/aws/_dev/build/docs/vpcflow.md b/test/packages/aws/_dev/build/docs/vpcflow.md index f707db5822..65e7e4fc9b 100644 --- a/test/packages/aws/_dev/build/docs/vpcflow.md +++ b/test/packages/aws/_dev/build/docs/vpcflow.md @@ -3,3 +3,5 @@ ## Logs {{fields "vpcflow"}} + +{{event "vpcflow"}} \ No newline at end of file diff --git a/test/packages/aws/_dev/build/docs/waf.md b/test/packages/aws/_dev/build/docs/waf.md new file mode 100644 index 0000000000..eeca8301b9 --- /dev/null +++ b/test/packages/aws/_dev/build/docs/waf.md @@ -0,0 +1,9 @@ +# waf + +## Logs + +The `waf` dataset is specifically for WAF logs. Export logs from Kinesis Data Firehose to Amazon S3 bucket which has SQS notification setup already. + +{{fields "waf"}} + +{{event "waf"}} \ No newline at end of file diff --git a/test/packages/aws/changelog.yml b/test/packages/aws/changelog.yml index 4ae28cb66a..72a1732365 100644 --- a/test/packages/aws/changelog.yml +++ b/test/packages/aws/changelog.yml @@ -1,21 +1,226 @@ # newer versions go on top - version: "999.999.999" changes: - - description: Use input groups. - type: enhancement # can be one of: enhancement, bugfix, breaking-change + - description: Change test public IPs to the supported subset + type: bugfix + link: https://github.com/elastic/integrations/pull/2327 +- version: "1.6.1" + changes: + - description: Fix the value of event.created in CloudTrail data stream. + type: enhancement + link: https://github.com/elastic/integrations/pull/2325 +- version: "1.6.0" + changes: + - description: Add max_number_of_messages config option to AWS S3 input config. + type: enhancement + link: https://github.com/elastic/integrations/pull/2299 +- version: "1.5.1" + changes: + - description: Add missing sample events + type: enhancement + link: https://github.com/elastic/integrations/pull/2282 +- version: "1.5.0" + changes: + - description: Support Kibana 8.0 + type: enhancement + link: https://github.com/elastic/integrations/pull/2206 +- version: "1.4.1" + changes: + - description: Add Overview dashboard for AWS S3 Storage Lens + type: enhancement + link: https://github.com/elastic/integrations/pull/2153 +- version: "1.4.0" + changes: + - description: Add integration for AWS S3 Storage Lens + type: enhancement + link: https://github.com/elastic/integrations/pull/2142 +- version: "1.3.2" + changes: + - description: Uniform with guidelines + type: enhancement + link: https://github.com/elastic/integrations/pull/2003 +- version: "1.3.1" + changes: + - description: Add config parameter descriptions + type: enhancement + link: https://github.com/elastic/integrations/pull/1917 +- version: "1.3.0" + changes: + - description: Add WAF datastream + type: enhancement + link: https://github.com/elastic/integrations/pull/1886 +- version: "1.2.2" + changes: + - description: Prevent pipeline script error + type: bugfix + link: https://github.com/elastic/integrations/pull/1871 +- version: "1.2.1" + changes: + - description: Fix logic that checks for the 'forwarded' tag + type: bugfix + link: https://github.com/elastic/integrations/pull/1797 +- version: "1.2.0" + changes: + - description: Update to ECS 1.12.0 + type: enhancement + link: https://github.com/elastic/integrations/pull/1687 +- version: "1.1.0" + changes: + - description: vpcflow sync with filebeat fileset + type: enhancement + link: https://github.com/elastic/integrations/pull/1772 +- version: "1.0.0" + changes: + - description: Release AWS as GA + type: enhancement + link: https://github.com/elastic/integrations/pull/1623 +- version: "0.10.7" + changes: + - description: Add proxy config + type: enhancement + link: https://github.com/elastic/integrations/pull/1648 +- version: "0.10.6" + changes: + - description: Fix aws.billing.EstimatedCharges field name + type: bugfix + link: https://github.com/elastic/integrations/pull/1606 +- version: "0.10.5" + changes: + - description: Add event.created field + type: bugfix + link: https://github.com/elastic/integrations/pull/1590 +- version: "0.10.4" + changes: + - description: Improve RDS dashboard + type: enhancement + link: https://github.com/elastic/integrations/pull/1449 +- version: "0.10.3" + changes: + - description: Convert to generated ECS fields + type: enhancement + link: https://github.com/elastic/integrations/pull/1465 +- version: '0.10.2' + changes: + - description: update to ECS 1.11.0 + type: enhancement + link: https://github.com/elastic/integrations/pull/1371 +- version: "0.10.1" + changes: + - description: Escape special characters in docs + type: enhancement + link: https://github.com/elastic/integrations/pull/1405 +- version: "0.10.0" + changes: + - description: Update integration description + type: enhancement + link: https://github.com/elastic/integrations/pull/1364 +- version: "0.9.3" + changes: + - description: Fix categories for each policy template + type: bugfix + link: https://github.com/elastic/integrations/pull/1357 +- version: "0.9.2" + changes: + - description: Add linked account information into billing metricset + type: enhancement + link: https://github.com/elastic/integrations/pull/1334 +- version: "0.9.1" + changes: + - description: Fix `aws.s3access` pipeline when remote IP is a `-` + type: bugfix + link: https://github.com/elastic/integrations/pull/1333 +- version: "0.9.0" + changes: + - description: Change default credential options to access keys + type: enhancement + link: https://github.com/elastic/integrations/pull/1320 +- version: "0.8.0" + changes: + - description: Set "event.module" and "event.dataset" + type: enhancement + link: https://github.com/elastic/integrations/pull/1245 +- version: "0.7.0" + changes: + - description: Introduce granularity using input_groups + type: enhancement link: https://github.com/elastic/integrations/pull/767 +- version: "0.6.4" + changes: + - description: Add support for Splunk authorization tokens + type: enhancement + link: https://github.com/elastic/integrations/pull/1147 +- version: "0.6.3" + changes: + - description: Fix bug in Third Party ingest pipeline + type: bugfix + link: https://github.com/elastic/integrations/pull/1201 +- version: "0.6.2" + changes: + - description: Removed incorrect `http.request.referrer` field from elb logs + type: bugfix + link: https://github.com/elastic/integrations/pull/1212 +- version: "0.6.1" + changes: + - description: Add support for CloudTrail Digest & Insight logs + type: enhancement + link: https://github.com/elastic/integrations/pull/1079 +- version: "0.6.0" + changes: + - description: Update ECS version, add event.original and preparing for package GA + type: enhancement + link: https://github.com/elastic/integrations/pull/1070 +- version: "0.5.6" + changes: + - description: Fix stack compatability + type: bugfix + link: https://github.com/elastic/integrations/pull/1000 +- version: "0.5.5" + changes: + - description: Allow role_arn work with access keys for AWS + type: enhancement + link: https://github.com/elastic/integrations/pull/979 +- version: "0.5.4" + changes: + - description: Rename s3 input to aws-s3. + type: enhancement + link: https://github.com/elastic/integrations/pull/631 +- version: "0.5.3" + changes: + - description: Add missing "geo" fields + type: enhancement + link: https://github.com/elastic/integrations/pull/919 +- version: "0.5.2" + changes: + - description: update to ECS 1.9.0 + type: enhancement + link: https://github.com/elastic/integrations/pull/834 +- version: "0.5.1" + changes: + - description: Ignore missing "json" field in ingest pipeline + type: bugfix + link: https://github.com/elastic/integrations/pull/791 +- version: "0.5.0" + changes: + - description: Moving edge processors to ingest pipeline + type: enhancement + link: https://github.com/elastic/integrations/pull/782 +- version: "0.4.2" + changes: + - description: Updating package owner + type: enhancement + link: https://github.com/elastic/integrations/pull/766 - version: "0.4.1" changes: - description: Correct sample event file. - type: bugfix # can be one of: enhancement, bugfix, breaking-change + type: bugfix link: https://github.com/elastic/integrations/pull/754 - version: "0.4.0" changes: - description: Add changes to use ECS 1.8 fields. - type: enhancement # can be one of: enhancement, bugfix, breaking-change + type: enhancement link: https://github.com/elastic/integrations/pull/721 - version: "0.0.3" changes: - description: initial release - type: enhancement # can be one of: enhancement, bugfix, breaking-change + type: enhancement link: https://github.com/elastic/integrations/pull/21 diff --git a/test/packages/aws/data_stream/billing/agent/stream/stream.yml.hbs b/test/packages/aws/data_stream/billing/agent/stream/stream.yml.hbs index caae1156d6..45017bd29c 100644 --- a/test/packages/aws/data_stream/billing/agent/stream/stream.yml.hbs +++ b/test/packages/aws/data_stream/billing/agent/stream/stream.yml.hbs @@ -32,4 +32,7 @@ cost_explorer_config.group_by_tag_keys: {{#each cost_explorer_config.group_by_tag_keys as |tag_key i|}} - {{tag_key}} {{/each}} +{{/if}} +{{#if proxy_url }} +proxy_url: {{proxy_url}} {{/if}} \ No newline at end of file diff --git a/test/packages/aws/data_stream/billing/fields/base-fields.yml b/test/packages/aws/data_stream/billing/fields/base-fields.yml index 7c798f4534..f3e0e8980d 100644 --- a/test/packages/aws/data_stream/billing/fields/base-fields.yml +++ b/test/packages/aws/data_stream/billing/fields/base-fields.yml @@ -10,3 +10,11 @@ - name: '@timestamp' type: date description: Event timestamp. +- name: event.module + type: constant_keyword + description: Event module + value: aws +- name: event.dataset + type: constant_keyword + description: Event dataset + value: aws.billing diff --git a/test/packages/aws/data_stream/billing/fields/ecs.yml b/test/packages/aws/data_stream/billing/fields/ecs.yml index a02d7269c5..83e3f6f122 100644 --- a/test/packages/aws/data_stream/billing/fields/ecs.yml +++ b/test/packages/aws/data_stream/billing/fields/ecs.yml @@ -1,53 +1,24 @@ -- name: cloud - title: Cloud - group: 2 - type: group - footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' - fields: - - name: account.id - level: extended - type: keyword - description: |- - The cloud account or organization id used to identify different entities in a multi-tenant environment. - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. - ignore_above: 1024 - - name: account.name - level: extended - type: keyword - description: |- - The cloud account name or alias used to identify different entities in a multi-tenant environment. - Examples: AWS account name, Google Cloud ORG display name. - ignore_above: 1024 - - name: availability_zone - level: extended - type: keyword - description: Availability zone in which this host is running. - ignore_above: 1024 - - name: instance.id - level: extended - type: keyword - description: Instance ID of the host machine. - ignore_above: 1024 - - name: machine.type - level: extended - type: keyword - description: Machine type of the host machine. - ignore_above: 1024 - - name: provider - level: extended - type: keyword - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - ignore_above: 1024 - - name: region - level: extended - type: keyword - description: Region in which this host is running. - ignore_above: 1024 -- name: ecs.version - type: keyword - description: ECS version this event conforms to. - example: 1.0.0 - ignore_above: 1024 -- name: service.type - type: keyword - description: Service type +- external: ecs + name: cloud +- external: ecs + name: cloud.account.id +- external: ecs + name: cloud.account.name +- external: ecs + name: cloud.availability_zone +- external: ecs + name: cloud.instance.id +- external: ecs + name: cloud.machine.type +- external: ecs + name: cloud.provider +- external: ecs + name: cloud.region +- external: ecs + name: ecs.version +- external: ecs + name: error +- external: ecs + name: error.message +- external: ecs + name: service.type diff --git a/test/packages/aws/data_stream/billing/fields/fields.yml b/test/packages/aws/data_stream/billing/fields/fields.yml index 5b3ee582a6..b91f8f3645 100644 --- a/test/packages/aws/data_stream/billing/fields/fields.yml +++ b/test/packages/aws/data_stream/billing/fields/fields.yml @@ -4,7 +4,7 @@ - name: billing type: group fields: - - name: EstimatedCharges.max + - name: EstimatedCharges type: long description: Maximum estimated charges for AWS acccount. - name: Currency diff --git a/test/packages/aws/data_stream/billing/fields/package-fields.yml b/test/packages/aws/data_stream/billing/fields/package-fields.yml index a8a7ee8dcc..7adc9facb0 100644 --- a/test/packages/aws/data_stream/billing/fields/package-fields.yml +++ b/test/packages/aws/data_stream/billing/fields/package-fields.yml @@ -17,3 +17,16 @@ type: object description: | Metrics that returned from Cloudwatch API query. + - name: linked_account + type: group + fields: + - name: id + type: keyword + description: > + ID used to identify linked account. + + - name: name + type: keyword + description: > + Name or alias used to identify linked account. + diff --git a/test/packages/aws/data_stream/billing/manifest.yml b/test/packages/aws/data_stream/billing/manifest.yml index e42030e46c..dcdfb390f1 100644 --- a/test/packages/aws/data_stream/billing/manifest.yml +++ b/test/packages/aws/data_stream/billing/manifest.yml @@ -1,4 +1,4 @@ -title: AWS billing metrics +title: AWS Billing Metrics release: beta type: metrics streams: @@ -27,6 +27,7 @@ streams: - "AZ" - "INSTANCE_TYPE" - "SERVICE" + - "LINKED_ACCOUNT" - name: cost_explorer_config.group_by_tag_keys type: text title: Cost Explorer Group By Tag Keys @@ -35,5 +36,5 @@ streams: show_user: true default: - "aws:createdBy" - title: AWS Billing metrics - description: Collect AWS billing metrics + title: AWS Billing Metrics + description: Collect billing metrics from Amazon Web Services with Elastic Agent. diff --git a/test/packages/aws/data_stream/billing/sample_event.json b/test/packages/aws/data_stream/billing/sample_event.json index 0a252492f0..832bb00231 100644 --- a/test/packages/aws/data_stream/billing/sample_event.json +++ b/test/packages/aws/data_stream/billing/sample_event.json @@ -22,17 +22,29 @@ }, "aws": { "billing": { - "metrics": { - "EstimatedCharges": { - "max": 1625.41 - } + "Currency": "USD", + "EstimatedCharges": 39.26, + "ServiceName": "AmazonEKS", + "AmortizedCost": { + "amount": 51.6, + "unit": "USD" + }, + "BlendedCost": { + "amount": 51.6, + "unit": "USD" + }, + "NormalizedUsageAmount": { + "amount": 672, + "unit": "N/A" + }, + "UnblendedCost": { + "amount": 51.6, + "unit": "USD" + }, + "UsageQuantity": { + "amount": 168, + "unit": "N/A" } - }, - "cloudwatch": { - "namespace": "AWS/Billing" - }, - "dimensions": { - "Currency": "USD" } }, "service": { diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-add-user-to-group-json.log-config.yml b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-add-user-to-group-json.log-config.yml deleted file mode 100644 index c39dc38617..0000000000 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-add-user-to-group-json.log-config.yml +++ /dev/null @@ -1,2 +0,0 @@ -dynamic_fields: - event.ingested: ".*" diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-add-user-to-group-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-add-user-to-group-json.log-expected.json index aefef7cf61..02145eb005 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-add-user-to-group-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-add-user-to-group-json.log-expected.json @@ -1,6 +1,13 @@ { "expected": [ { + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "tags": [ + "preserve_original_event" + ], "cloud": { "region": "us-east-2", "account": { @@ -8,21 +15,19 @@ } }, "@timestamp": "2014-03-25T21:08:14.000Z", + "ecs": { + "version": "1.12.0" + }, "related": { "user": [ "Alice", "Bob" ] }, - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, "event": { - "ingested": "2021-04-23T12:15:54.290352893Z", "original": "{\"eventVersion\":\"1.0\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EX_PRINCIPAL_ID\",\"arn\":\"arn:aws:iam::123456789012:user/Alice\",\"accountId\":\"123456789012\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"false\",\"creationDate\":\"2014-03-25T18:45:11Z\"}}},\"eventTime\":\"2014-03-25T21:08:14Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"AddUserToGroup\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"AWSConsole\",\"requestParameters\":{\"userName\":\"Bob\",\"groupName\":\"admin\"},\"responseElements\":null}", "provider": "iam.amazonaws.com", - "created": "2014-03-25T21:08:14.000Z", + "created": "2021-11-11T01:02:03.123456789Z", "kind": "event", "action": "AddUserToGroup", "type": [ diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-assume-role-json.log b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-assume-role-json.log index c2a4a5e884..90e496fc0f 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-assume-role-json.log +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-assume-role-json.log @@ -1 +1 @@ -{"eventVersion":"1.05","userIdentity":{"type":"AssumedRole","principalId":"AROAIN5ATK5U7KEXAMPLE:JohnRole1","arn":"arn:aws:sts::111111111111:assumed-role/JohnDoe/JohnRole1","accountId":"111111111111","accessKeyId":"AKIAI44QH8DHBEXAMPLE","sessionContext":{"attributes":{"mfaAuthenticated":"false","creationDate":"2019-10-02T21:50:54Z"},"sessionIssuer":{"type":"Role","principalId":"AROAIN5ATK5U7KEXAMPLE","arn":"arn:aws:iam::111111111111:role/JohnRole1","accountId":"111111111111","userName":"JohnDoe"}}},"eventTime":"2019-10-02T22:12:29Z","eventSource":"sts.amazonaws.com","eventName":"AssumeRole","awsRegion":"us-east-2","sourceIPAddress":"123.145.67.89","userAgent":"aws-cli/1.16.248 Python/3.4.7 Linux/4.9.184-0.1.ac.235.83.329.metal1.x86_64 botocore/1.12.239","requestParameters":{"incomingTransitiveTags":{"Department":"Engineering"},"tags":[{"value":"johndoe@example.com","key":"Email"},{"value":"12345","key":"CostCenter"}],"roleArn":"arn:aws:iam::111111111111:role/JohnRole2","roleSessionName":"Role2WithTags","transitiveTagKeys":["Email","CostCenter"],"durationSeconds":3600},"responseElements":{"credentials":{"accessKeyId":"ASIAWHOJDLGPOEXAMPLE","expiration":"Oct 2, 2019 11:12:29 PM","sessionToken":"AgoJb3JpZ2luX2VjEB4aCXVzLXdlc3QtMSJHMEXAMPLETOKEN+//rJb8Lo30mFc5MlhFCEbubZvEj0wHB/mDMwIgSEe9gk/Zjr09tZV7F1HDTMhmEXAMPLETOKEN/iEJ/rkqngII9///////////ARABGgw0MjgzMDc4NjM5NjYiDLZjZFKwP4qxQG5sFCryASO4UPz5qE97wPPH1eLMvs7CgSDBSWfonmRTCfokm2FN1+hWUdQQH6adjbbrVLFL8c3jSsBhQ383AvxpwK5YRuDE1AI/+C+WKFZb701eiv9J5La2EXAMPLETOKEN/c7S5Iro1WUJ0q3Cxuo/8HUoSxVhQHM7zF7mWWLhXLEQ52ivL+F6q5dpXu4aTFedpMfnJa8JtkWwG9x1Axj0Ypy2ok8v5unpQGWych1vwdvj6ez1Dm8Xg1+qIzXILiEXAMPLETOKEN/vQGqu8H+nxp3kabcrtOvTFTvxX6vsc8OGwUfHhzAfYGEXAMPLETOKEN/L6v1yMM3B1OwFOrQBno1HEjf1oNI8RnQiMNFdUOtwYj7HUZIOCZmjfN8PPHq77N7GJl9lzvIZKQA0Owcjg+mc78zHCj8y0siY8C96paEXAMPLETOKEN/E3cpksxWdgs91HRzJWScjN2+r2LTGjYhyPqcmFzzo2mCE7mBNEXAMPLETOKEN/oJy+2o83YNW5tOiDmczgDzJZ4UKR84yGYOMfSnF4XcEJrDgAJ3OJFwmTcTQICAlSwLEXAMPLETOKEN"},"assumedRoleUser":{"assumedRoleId":"AROAIFR7WHDTSOYQYHFUE:Role2WithTags","arn":"arn:aws:sts::111111111111:assumed-role/test-role/Role2WithTags"}},"requestID":"b96b0e4e-e561-11e9-8b3f-7b396EXAMPLE","eventID":"1917948f-3042-46ec-98e2-62865EXAMPLE","resources":[{"ARN":"arn:aws:iam::111122223333:role/JohnRole2","accountId":"111111111111","type":"AWS::IAM::Role"}],"eventType":"AwsApiCall","recipientAccountId":"111111111111"} +{"eventVersion":"1.05","userIdentity":{"type":"AssumedRole","principalId":"AROAIN5ATK5U7KEXAMPLE:JohnRole1","arn":"arn:aws:sts::111111111111:assumed-role/JohnDoe/JohnRole1","accountId":"111111111111","accessKeyId":"AKIAI44QH8DHBEXAMPLE","sessionContext":{"attributes":{"mfaAuthenticated":"false","creationDate":"2019-10-02T21:50:54Z"},"sessionIssuer":{"type":"Role","principalId":"AROAIN5ATK5U7KEXAMPLE","arn":"arn:aws:iam::111111111111:role/JohnRole1","accountId":"111111111111","userName":"JohnDoe"}}},"eventTime":"2019-10-02T22:12:29Z","eventSource":"sts.amazonaws.com","eventName":"AssumeRole","awsRegion":"us-east-2","sourceIPAddress":"81.2.69.144","userAgent":"aws-cli/1.16.248 Python/3.4.7 Linux/4.9.184-0.1.ac.235.83.329.metal1.x86_64 botocore/1.12.239","requestParameters":{"incomingTransitiveTags":{"Department":"Engineering"},"tags":[{"value":"johndoe@example.com","key":"Email"},{"value":"12345","key":"CostCenter"}],"roleArn":"arn:aws:iam::111111111111:role/JohnRole2","roleSessionName":"Role2WithTags","transitiveTagKeys":["Email","CostCenter"],"durationSeconds":3600},"responseElements":{"credentials":{"accessKeyId":"ASIAWHOJDLGPOEXAMPLE","expiration":"Oct 2, 2019 11:12:29 PM","sessionToken":"AgoJb3JpZ2luX2VjEB4aCXVzLXdlc3QtMSJHMEXAMPLETOKEN+//rJb8Lo30mFc5MlhFCEbubZvEj0wHB/mDMwIgSEe9gk/Zjr09tZV7F1HDTMhmEXAMPLETOKEN/iEJ/rkqngII9///////////ARABGgw0MjgzMDc4NjM5NjYiDLZjZFKwP4qxQG5sFCryASO4UPz5qE97wPPH1eLMvs7CgSDBSWfonmRTCfokm2FN1+hWUdQQH6adjbbrVLFL8c3jSsBhQ383AvxpwK5YRuDE1AI/+C+WKFZb701eiv9J5La2EXAMPLETOKEN/c7S5Iro1WUJ0q3Cxuo/8HUoSxVhQHM7zF7mWWLhXLEQ52ivL+F6q5dpXu4aTFedpMfnJa8JtkWwG9x1Axj0Ypy2ok8v5unpQGWych1vwdvj6ez1Dm8Xg1+qIzXILiEXAMPLETOKEN/vQGqu8H+nxp3kabcrtOvTFTvxX6vsc8OGwUfHhzAfYGEXAMPLETOKEN/L6v1yMM3B1OwFOrQBno1HEjf1oNI8RnQiMNFdUOtwYj7HUZIOCZmjfN8PPHq77N7GJl9lzvIZKQA0Owcjg+mc78zHCj8y0siY8C96paEXAMPLETOKEN/E3cpksxWdgs91HRzJWScjN2+r2LTGjYhyPqcmFzzo2mCE7mBNEXAMPLETOKEN/oJy+2o83YNW5tOiDmczgDzJZ4UKR84yGYOMfSnF4XcEJrDgAJ3OJFwmTcTQICAlSwLEXAMPLETOKEN"},"assumedRoleUser":{"assumedRoleId":"AROAIFR7WHDTSOYQYHFUE:Role2WithTags","arn":"arn:aws:sts::111111111111:assumed-role/test-role/Role2WithTags"}},"requestID":"b96b0e4e-e561-11e9-8b3f-7b396EXAMPLE","eventID":"1917948f-3042-46ec-98e2-62865EXAMPLE","resources":[{"ARN":"arn:aws:iam::111122223333:role/JohnRole2","accountId":"111111111111","type":"AWS::IAM::Role"}],"eventType":"AwsApiCall","recipientAccountId":"111111111111"} diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-assume-role-json.log-config.yml b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-assume-role-json.log-config.yml deleted file mode 100644 index c39dc38617..0000000000 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-assume-role-json.log-config.yml +++ /dev/null @@ -1,2 +0,0 @@ -dynamic_fields: - event.ingested: ".*" diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-assume-role-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-assume-role-json.log-expected.json index 4b2156077d..6231a4dd28 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-assume-role-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-assume-role-json.log-expected.json @@ -8,32 +8,35 @@ } }, "@timestamp": "2019-10-02T22:12:29.000Z", + "ecs": { + "version": "1.12.0" + }, "source": { "geo": { - "continent_name": "Asia", - "region_iso_code": "CN-CQ", - "country_name": "China", - "region_name": "Chongqing", + "continent_name": "Europe", + "region_iso_code": "GB-OXF", + "city_name": "Abingdon", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "region_name": "Oxfordshire", "location": { - "lon": 106.5531, - "lat": 29.5569 - }, - "country_iso_code": "CN" + "lon": -1.3614, + "lat": 51.7095 + } }, "as": { - "number": 4837, + "number": 20712, "organization": { - "name": "CHINA UNICOM China169 Backbone" + "name": "Andrews \u0026 Arnold Ltd" } }, - "address": "123.145.67.89", - "ip": "123.145.67.89" + "address": "81.2.69.144", + "ip": "81.2.69.144" }, "event": { - "ingested": "2021-04-23T12:15:54.325928891Z", - "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"AssumedRole\",\"principalId\":\"AROAIN5ATK5U7KEXAMPLE:JohnRole1\",\"arn\":\"arn:aws:sts::111111111111:assumed-role/JohnDoe/JohnRole1\",\"accountId\":\"111111111111\",\"accessKeyId\":\"AKIAI44QH8DHBEXAMPLE\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"false\",\"creationDate\":\"2019-10-02T21:50:54Z\"},\"sessionIssuer\":{\"type\":\"Role\",\"principalId\":\"AROAIN5ATK5U7KEXAMPLE\",\"arn\":\"arn:aws:iam::111111111111:role/JohnRole1\",\"accountId\":\"111111111111\",\"userName\":\"JohnDoe\"}}},\"eventTime\":\"2019-10-02T22:12:29Z\",\"eventSource\":\"sts.amazonaws.com\",\"eventName\":\"AssumeRole\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"123.145.67.89\",\"userAgent\":\"aws-cli/1.16.248 Python/3.4.7 Linux/4.9.184-0.1.ac.235.83.329.metal1.x86_64 botocore/1.12.239\",\"requestParameters\":{\"incomingTransitiveTags\":{\"Department\":\"Engineering\"},\"tags\":[{\"value\":\"johndoe@example.com\",\"key\":\"Email\"},{\"value\":\"12345\",\"key\":\"CostCenter\"}],\"roleArn\":\"arn:aws:iam::111111111111:role/JohnRole2\",\"roleSessionName\":\"Role2WithTags\",\"transitiveTagKeys\":[\"Email\",\"CostCenter\"],\"durationSeconds\":3600},\"responseElements\":{\"credentials\":{\"accessKeyId\":\"ASIAWHOJDLGPOEXAMPLE\",\"expiration\":\"Oct 2, 2019 11:12:29 PM\",\"sessionToken\":\"AgoJb3JpZ2luX2VjEB4aCXVzLXdlc3QtMSJHMEXAMPLETOKEN+//rJb8Lo30mFc5MlhFCEbubZvEj0wHB/mDMwIgSEe9gk/Zjr09tZV7F1HDTMhmEXAMPLETOKEN/iEJ/rkqngII9///////////ARABGgw0MjgzMDc4NjM5NjYiDLZjZFKwP4qxQG5sFCryASO4UPz5qE97wPPH1eLMvs7CgSDBSWfonmRTCfokm2FN1+hWUdQQH6adjbbrVLFL8c3jSsBhQ383AvxpwK5YRuDE1AI/+C+WKFZb701eiv9J5La2EXAMPLETOKEN/c7S5Iro1WUJ0q3Cxuo/8HUoSxVhQHM7zF7mWWLhXLEQ52ivL+F6q5dpXu4aTFedpMfnJa8JtkWwG9x1Axj0Ypy2ok8v5unpQGWych1vwdvj6ez1Dm8Xg1+qIzXILiEXAMPLETOKEN/vQGqu8H+nxp3kabcrtOvTFTvxX6vsc8OGwUfHhzAfYGEXAMPLETOKEN/L6v1yMM3B1OwFOrQBno1HEjf1oNI8RnQiMNFdUOtwYj7HUZIOCZmjfN8PPHq77N7GJl9lzvIZKQA0Owcjg+mc78zHCj8y0siY8C96paEXAMPLETOKEN/E3cpksxWdgs91HRzJWScjN2+r2LTGjYhyPqcmFzzo2mCE7mBNEXAMPLETOKEN/oJy+2o83YNW5tOiDmczgDzJZ4UKR84yGYOMfSnF4XcEJrDgAJ3OJFwmTcTQICAlSwLEXAMPLETOKEN\"},\"assumedRoleUser\":{\"assumedRoleId\":\"AROAIFR7WHDTSOYQYHFUE:Role2WithTags\",\"arn\":\"arn:aws:sts::111111111111:assumed-role/test-role/Role2WithTags\"}},\"requestID\":\"b96b0e4e-e561-11e9-8b3f-7b396EXAMPLE\",\"eventID\":\"1917948f-3042-46ec-98e2-62865EXAMPLE\",\"resources\":[{\"ARN\":\"arn:aws:iam::111122223333:role/JohnRole2\",\"accountId\":\"111111111111\",\"type\":\"AWS::IAM::Role\"}],\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"111111111111\"}", + "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"AssumedRole\",\"principalId\":\"AROAIN5ATK5U7KEXAMPLE:JohnRole1\",\"arn\":\"arn:aws:sts::111111111111:assumed-role/JohnDoe/JohnRole1\",\"accountId\":\"111111111111\",\"accessKeyId\":\"AKIAI44QH8DHBEXAMPLE\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"false\",\"creationDate\":\"2019-10-02T21:50:54Z\"},\"sessionIssuer\":{\"type\":\"Role\",\"principalId\":\"AROAIN5ATK5U7KEXAMPLE\",\"arn\":\"arn:aws:iam::111111111111:role/JohnRole1\",\"accountId\":\"111111111111\",\"userName\":\"JohnDoe\"}}},\"eventTime\":\"2019-10-02T22:12:29Z\",\"eventSource\":\"sts.amazonaws.com\",\"eventName\":\"AssumeRole\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"81.2.69.144\",\"userAgent\":\"aws-cli/1.16.248 Python/3.4.7 Linux/4.9.184-0.1.ac.235.83.329.metal1.x86_64 botocore/1.12.239\",\"requestParameters\":{\"incomingTransitiveTags\":{\"Department\":\"Engineering\"},\"tags\":[{\"value\":\"johndoe@example.com\",\"key\":\"Email\"},{\"value\":\"12345\",\"key\":\"CostCenter\"}],\"roleArn\":\"arn:aws:iam::111111111111:role/JohnRole2\",\"roleSessionName\":\"Role2WithTags\",\"transitiveTagKeys\":[\"Email\",\"CostCenter\"],\"durationSeconds\":3600},\"responseElements\":{\"credentials\":{\"accessKeyId\":\"ASIAWHOJDLGPOEXAMPLE\",\"expiration\":\"Oct 2, 2019 11:12:29 PM\",\"sessionToken\":\"AgoJb3JpZ2luX2VjEB4aCXVzLXdlc3QtMSJHMEXAMPLETOKEN+//rJb8Lo30mFc5MlhFCEbubZvEj0wHB/mDMwIgSEe9gk/Zjr09tZV7F1HDTMhmEXAMPLETOKEN/iEJ/rkqngII9///////////ARABGgw0MjgzMDc4NjM5NjYiDLZjZFKwP4qxQG5sFCryASO4UPz5qE97wPPH1eLMvs7CgSDBSWfonmRTCfokm2FN1+hWUdQQH6adjbbrVLFL8c3jSsBhQ383AvxpwK5YRuDE1AI/+C+WKFZb701eiv9J5La2EXAMPLETOKEN/c7S5Iro1WUJ0q3Cxuo/8HUoSxVhQHM7zF7mWWLhXLEQ52ivL+F6q5dpXu4aTFedpMfnJa8JtkWwG9x1Axj0Ypy2ok8v5unpQGWych1vwdvj6ez1Dm8Xg1+qIzXILiEXAMPLETOKEN/vQGqu8H+nxp3kabcrtOvTFTvxX6vsc8OGwUfHhzAfYGEXAMPLETOKEN/L6v1yMM3B1OwFOrQBno1HEjf1oNI8RnQiMNFdUOtwYj7HUZIOCZmjfN8PPHq77N7GJl9lzvIZKQA0Owcjg+mc78zHCj8y0siY8C96paEXAMPLETOKEN/E3cpksxWdgs91HRzJWScjN2+r2LTGjYhyPqcmFzzo2mCE7mBNEXAMPLETOKEN/oJy+2o83YNW5tOiDmczgDzJZ4UKR84yGYOMfSnF4XcEJrDgAJ3OJFwmTcTQICAlSwLEXAMPLETOKEN\"},\"assumedRoleUser\":{\"assumedRoleId\":\"AROAIFR7WHDTSOYQYHFUE:Role2WithTags\",\"arn\":\"arn:aws:sts::111111111111:assumed-role/test-role/Role2WithTags\"}},\"requestID\":\"b96b0e4e-e561-11e9-8b3f-7b396EXAMPLE\",\"eventID\":\"1917948f-3042-46ec-98e2-62865EXAMPLE\",\"resources\":[{\"ARN\":\"arn:aws:iam::111122223333:role/JohnRole2\",\"accountId\":\"111111111111\",\"type\":\"AWS::IAM::Role\"}],\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"111111111111\"}", "provider": "sts.amazonaws.com", - "created": "2019-10-02T22:12:29.000Z", + "created": "2021-11-11T01:02:03.123456789Z", "kind": "event", "action": "AssumeRole", "id": "1917948f-3042-46ec-98e2-62865EXAMPLE", @@ -120,7 +123,10 @@ "name": "Spider" }, "version": "1.16.248" - } + }, + "tags": [ + "preserve_original_event" + ] } ] } \ No newline at end of file diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-change-password-json.log-config.yml b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-change-password-json.log-config.yml deleted file mode 100644 index c39dc38617..0000000000 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-change-password-json.log-config.yml +++ /dev/null @@ -1,2 +0,0 @@ -dynamic_fields: - event.ingested: ".*" diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-change-password-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-change-password-json.log-expected.json index 69673ce6bb..9dcab5261a 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-change-password-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-change-password-json.log-expected.json @@ -1,6 +1,13 @@ { "expected": [ { + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "tags": [ + "preserve_original_event" + ], "cloud": { "region": "us-east-1", "account": { @@ -8,20 +15,18 @@ } }, "@timestamp": "2020-01-09T00:09:33.000Z", + "ecs": { + "version": "1.12.0" + }, "related": { "user": [ "Alice" ] }, - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, "event": { - "ingested": "2021-04-23T12:15:54.376665102Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"0123456789012\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\"},\"eventTime\":\"2020-01-09T00:09:33Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"ChangePassword\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46\",\"errorCode\":\"AccessDeniedException\",\"errorMessage\":\"An unknown error occurred\",\"requestParameters\":null,\"responseElements\":null,\"requestID\":\"EXAMPLE-5204-4fed-9c60-9c6EXAMPLE\",\"eventID\":\"EXAMPLE-b92f-48bb-8c4c-efeEXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", - "created": "2020-01-09T00:09:33.000Z", + "created": "2021-11-11T01:02:03.123456789Z", "kind": "event", "action": "ChangePassword", "id": "EXAMPLE-b92f-48bb-8c4c-efeEXAMPLE", @@ -63,6 +68,13 @@ } }, { + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "tags": [ + "preserve_original_event" + ], "cloud": { "region": "us-east-1", "account": { @@ -70,20 +82,18 @@ } }, "@timestamp": "2020-01-09T00:03:36.000Z", + "ecs": { + "version": "1.12.0" + }, "related": { "user": [ "Alice" ] }, - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, "event": { - "ingested": "2021-04-23T12:15:54.376673279Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"0123456789012\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\"},\"eventTime\":\"2020-01-09T00:03:36Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"ChangePassword\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46\",\"requestParameters\":null,\"responseElements\":null,\"requestID\":\"EXAMPLE-5c16-4eda-9724-EXAMPLE\",\"eventID\":\"EXAMPLE-35a7-4c25-9fc7-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", - "created": "2020-01-09T00:03:36.000Z", + "created": "2021-11-11T01:02:03.123456789Z", "kind": "event", "action": "ChangePassword", "id": "EXAMPLE-35a7-4c25-9fc7-EXAMPLE", diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-cloudtrail-digest-json.log-config.yml b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-cloudtrail-digest-json.log-config.yml deleted file mode 100644 index c39dc38617..0000000000 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-cloudtrail-digest-json.log-config.yml +++ /dev/null @@ -1,2 +0,0 @@ -dynamic_fields: - event.ingested: ".*" diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-cloudtrail-digest-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-cloudtrail-digest-json.log-expected.json index 00bd8dda0e..0c1cc9a2c9 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-cloudtrail-digest-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-cloudtrail-digest-json.log-expected.json @@ -6,23 +6,27 @@ "id": "123456789123" } }, + "@timestamp": "2020-09-11T19:36:49.000Z", "file": { "path": "AWSLogs/123456789123/CloudTrail-Digest/us-west-2/2020/09/11/123456789123_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T183649Z.json.gz", "hash": { "sha256": "10e0872f32fa1d299d0cc98e94d4c88a6a2eada9d9fc3ae6d53dfe8d54c7caf807072f1e1eec47efdeecfcc22483887f8fddfc954ae587fba43e7676b5547f432fa8722ba1c5baa6b233bcb528ce7c01e3748aab8f28c16c024de79da820128b4c9e5ce65e98a9c4e631687ecc89c224a11bb3df06ce441ff740e4ac9fbd41159e77f5863550118284121f193e357866fbd0463faffb56e194af196e35a7675c3bbd0a398f43159343c3f59129d6339a281a8fdb3192f3fffea9bd21dbb0a705ebfae1921f2133aab0ad29522aea6df0828c1780d3f3ed6b8270ab3ba24459916b0fbbe82fba6ff9677bafe7306e0f5edcc0f1508cdb4e36f3e3b30e653e9987" } }, - "@timestamp": "2020-09-11T19:36:49.000Z", + "ecs": { + "version": "1.12.0" + }, "related": { "hash": [ "10e0872f32fa1d299d0cc98e94d4c88a6a2eada9d9fc3ae6d53dfe8d54c7caf807072f1e1eec47efdeecfcc22483887f8fddfc954ae587fba43e7676b5547f432fa8722ba1c5baa6b233bcb528ce7c01e3748aab8f28c16c024de79da820128b4c9e5ce65e98a9c4e631687ecc89c224a11bb3df06ce441ff740e4ac9fbd41159e77f5863550118284121f193e357866fbd0463faffb56e194af196e35a7675c3bbd0a398f43159343c3f59129d6339a281a8fdb3192f3fffea9bd21dbb0a705ebfae1921f2133aab0ad29522aea6df0828c1780d3f3ed6b8270ab3ba24459916b0fbbe82fba6ff9677bafe7306e0f5edcc0f1508cdb4e36f3e3b30e653e9987" ] }, "event": { - "ingested": "2021-04-23T12:15:54.447816418Z", "original": "{\"awsAccountId\":\"123456789123\",\"digestStartTime\":\"2020-09-11T18:36:49Z\",\"digestEndTime\":\"2020-09-11T19:36:49Z\",\"digestS3Bucket\":\"alice-bucket\",\"digestS3Object\":\"AWSLogs/123456789123/CloudTrail-Digest/us-west-2/2020/09/11/123456789123_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T193649Z.json.gz\",\"digestPublicKeyFingerprint\":\"47aaa19f7eec22e9bd0b5e58cfade8cb\",\"digestSignatureAlgorithm\":\"SHA256withRSA\",\"newestEventTime\":\"2020-09-11T19:26:24Z\",\"oldestEventTime\":\"2020-09-11T18:32:04Z\",\"previousDigestS3Bucket\":\"alice-bucket\",\"previousDigestS3Object\":\"AWSLogs/123456789123/CloudTrail-Digest/us-west-2/2020/09/11/123456789123_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T183649Z.json.gz\",\"previousDigestHashValue\":\"531914fcfa0dbacf0c9dd1475a1fdcb5dea6e85921409f3c3ec0ba39063c860\",\"previousDigestHashAlgorithm\":\"SHA-256\",\"previousDigestSignature\":\"10e0872f32fa1d299d0cc98e94d4c88a6a2eada9d9fc3ae6d53dfe8d54c7caf807072f1e1eec47efdeecfcc22483887f8fddfc954ae587fba43e7676b5547f432fa8722ba1c5baa6b233bcb528ce7c01e3748aab8f28c16c024de79da820128b4c9e5ce65e98a9c4e631687ecc89c224a11bb3df06ce441ff740e4ac9fbd41159e77f5863550118284121f193e357866fbd0463faffb56e194af196e35a7675c3bbd0a398f43159343c3f59129d6339a281a8fdb3192f3fffea9bd21dbb0a705ebfae1921f2133aab0ad29522aea6df0828c1780d3f3ed6b8270ab3ba24459916b0fbbe82fba6ff9677bafe7306e0f5edcc0f1508cdb4e36f3e3b30e653e9987\",\"logFiles\":[{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1930Z_l2pGqVS53QcGdAkp.json.gz\",\"hashValue\":\"420784a5bbc12e9ac442451e8ec1356744fdeabf4fee0d2222508db6d448139c\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:26:24Z\",\"oldestEventTime\":\"2020-09-11T19:26:24Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1915Z_TIKlbLnJ6IwUxqxw.json.gz\",\"hashValue\":\"4e1eb2a8b41d032cbb16e5449fc8f3eac304e7d43017a391b37c788c77336196\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:11:18Z\",\"oldestEventTime\":\"2020-09-11T19:11:18Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1835Z_OPJhVNodH1gY760s.json.gz\",\"hashValue\":\"2695aeb3b4c1f021fe76e0b36f5ac15e557c41c58af6eef282d77ef056210d70\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:32:04Z\",\"oldestEventTime\":\"2020-09-11T18:32:04Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1925Z_zJNGzQovyNAImZV9.json.gz\",\"hashValue\":\"45a2906f55cbfc912584e9425f8d3d8d6fabf571a45a5ecd7d2a0f4132b81689\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:21:28Z\",\"oldestEventTime\":\"2020-09-11T19:21:28Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1855Z_RqN9YzoKAJCKbejj.json.gz\",\"hashValue\":\"515cc8be750d815266b4fc799c7600765f22502d29f5bb9d5c8969ffc5ab7097\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:51:21Z\",\"oldestEventTime\":\"2020-09-11T18:51:21Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1850Z_jLldN7U8XrspES8p.json.gz\",\"hashValue\":\"18650414e79e084dff02da66253f071347f7bb5c4863279bafe7762a980f7c0b\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:46:45Z\",\"oldestEventTime\":\"2020-09-11T18:46:45Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1905Z_jBNdmg4bSGxZ3wC8.json.gz\",\"hashValue\":\"54050ec665636f1985f5b51ae43c74a58282cb2e500492a45f20a4dc1bf8a6d5\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:01:06Z\",\"oldestEventTime\":\"2020-09-11T19:01:06Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1920Z_bj5DRrmILF6jK23a.json.gz\",\"hashValue\":\"6e0d8fcbd712d3f6d1caf4a872681f4290b05ed8a8f1c9450a0a6db92ccab4d7\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:16:12Z\",\"oldestEventTime\":\"2020-09-11T19:16:12Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1900Z_6LjrkrhsLQMzCiSN.json.gz\",\"hashValue\":\"b2b0e2804d1c6b92d76eee203d7eba32d3d003e6967f175723a83ecc2d7ad4ba\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:56:05Z\",\"oldestEventTime\":\"2020-09-11T18:56:05Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1910Z_DLyqye8LaeoD204N.json.gz\",\"hashValue\":\"4397a13565a67d9ed6e57737b98eb7e61ca52bb191c9b5da0423136dfc5581c7\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:06:31Z\",\"oldestEventTime\":\"2020-09-11T19:06:31Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1845Z_TSDKyASOn2ejOq5n.json.gz\",\"hashValue\":\"94f09d2398632c7b0c0066ed5d56768632dd2e06ed9c80af9d0c2c5f59bd60b6\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:41:58Z\",\"oldestEventTime\":\"2020-09-11T18:41:58Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1840Z_btJydJ2t7hCRnjsN.json.gz\",\"hashValue\":\"9044f9a05d70688bc6f6048d5f8d00764ab65e132b8ffefb193b22ca4394d771\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:37:10Z\",\"oldestEventTime\":\"2020-09-11T18:37:10Z\"}]}", "type": "info", - "kind": "event" + "created": "2021-11-11T01:02:03.123456789Z", + "kind": "event", + "outcome": "success" }, "aws": { "cloudtrail": { @@ -137,7 +141,10 @@ } } } - } + }, + "tags": [ + "preserve_original_event" + ] } ] } \ No newline at end of file diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-common-config.yml b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 0000000000..e8c22fb0e2 --- /dev/null +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,7 @@ +dynamic_fields: + event.ingested: ".*" +fields: + # Simulate @timestamp value from Filebeat. + '@timestamp': '2021-11-11T01:02:03.123456789Z' + tags: + - preserve_original_event diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-console-login-json.log b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-console-login-json.log index 14fb436a93..315e72e609 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-console-login-json.log +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-console-login-json.log @@ -1,3 +1,3 @@ -{"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"AIDACKCEVSQ6C2EXAMPLE","arn":"arn:aws:iam::111122223333:user/JohnDoe","accountId":"111122223333","userName":"JohnDoe"},"eventTime":"2014-07-16T15:49:27Z","eventSource":"signin.amazonaws.com","eventName":"ConsoleLogin","awsRegion":"us-east-2","sourceIPAddress":"192.0.2.110","userAgent":"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0","requestParameters":null,"responseElements":{"ConsoleLogin":"Success"},"additionalEventData":{"MobileVersion":"No","LoginTo":"https://console.aws.amazon.com/s3/","MFAUsed":"No"},"eventID":"3fcfb182-98f8-4744-bd45-10aEXAMPLE"} -{"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"AIDACKCEVSQ6C2EXAMPLE","arn":"arn:aws:iam::111122223333:user/JaneDoe","accountId":"111122223333","userName":"JaneDoe"},"eventTime":"2014-07-08T17:35:27Z","eventSource":"signin.amazonaws.com","eventName":"ConsoleLogin","awsRegion":"us-east-2","sourceIPAddress":"192.0.2.100","userAgent":"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0","errorMessage":"Failed authentication","requestParameters":null,"responseElements":{"ConsoleLogin":"Failure"},"additionalEventData":{"MobileVersion":"No","LoginTo":"https://console.aws.amazon.com/sns","MFAUsed":"No"},"eventID":"11ea990b-4678-4bcd-8fbe-625EXAMPLE"} -{"eventVersion":"1.05","userIdentity":{"type":"AssumedRole","principalId":"AROAIDPPEZS35WEXAMPLE:AssumedRoleSessionName","arn":"arn:aws:sts::123456789012:assumed-role/RoleToBeAssumed/MySessionName","accountId":"123456789012","accessKeyId":"AKIAIOSFODNN7EXAMPLE","sessionContext":{"attributes":{"mfaAuthenticated":"false","creationDate":"20131102T010628Z"},"sessionIssuer":{"type":"Role","principalId":"AROAIDPPEZS35WEXAMPLE","arn":"arn:aws:iam::123456789012:role/RoleToBeAssumed","accountId":"123456789012","userName":"RoleToBeAssumed"}}},"eventTime":"2014-07-08T17:35:27Z","eventSource":"signin.amazonaws.com","eventName":"ConsoleLogin","awsRegion":"us-east-2","sourceIPAddress":"192.0.2.100","userAgent":"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0","errorMessage":"Failed authentication","requestParameters":null,"responseElements":{"ConsoleLogin":"Failure"},"additionalEventData":{"MobileVersion":"No","LoginTo":"https://console.aws.amazon.com/sns","MFAUsed":"No"},"eventID":"11ea990b-4678-4bcd-8fbe-625EXAMPLE"} +{"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"AIDACKCEVSQ6C2EXAMPLE","arn":"arn:aws:iam::111122223333:user/JohnDoe","accountId":"111122223333","userName":"JohnDoe"},"eventTime":"2014-07-16T15:49:27Z","eventSource":"signin.amazonaws.com","eventName":"ConsoleLogin","awsRegion":"us-east-2","sourceIPAddress":"89.160.20.156","userAgent":"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0","requestParameters":null,"responseElements":{"ConsoleLogin":"Success"},"additionalEventData":{"MobileVersion":"No","LoginTo":"https://console.aws.amazon.com/s3/","MFAUsed":"No"},"eventID":"3fcfb182-98f8-4744-bd45-10aEXAMPLE"} +{"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"AIDACKCEVSQ6C2EXAMPLE","arn":"arn:aws:iam::111122223333:user/JaneDoe","accountId":"111122223333","userName":"JaneDoe"},"eventTime":"2014-07-08T17:35:27Z","eventSource":"signin.amazonaws.com","eventName":"ConsoleLogin","awsRegion":"us-east-2","sourceIPAddress":"89.160.20.156","userAgent":"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0","errorMessage":"Failed authentication","requestParameters":null,"responseElements":{"ConsoleLogin":"Failure"},"additionalEventData":{"MobileVersion":"No","LoginTo":"https://console.aws.amazon.com/sns","MFAUsed":"No"},"eventID":"11ea990b-4678-4bcd-8fbe-625EXAMPLE"} +{"eventVersion":"1.05","userIdentity":{"type":"AssumedRole","principalId":"AROAIDPPEZS35WEXAMPLE:AssumedRoleSessionName","arn":"arn:aws:sts::123456789012:assumed-role/RoleToBeAssumed/MySessionName","accountId":"123456789012","accessKeyId":"AKIAIOSFODNN7EXAMPLE","sessionContext":{"attributes":{"mfaAuthenticated":"false","creationDate":"20131102T010628Z"},"sessionIssuer":{"type":"Role","principalId":"AROAIDPPEZS35WEXAMPLE","arn":"arn:aws:iam::123456789012:role/RoleToBeAssumed","accountId":"123456789012","userName":"RoleToBeAssumed"}}},"eventTime":"2014-07-08T17:35:27Z","eventSource":"signin.amazonaws.com","eventName":"ConsoleLogin","awsRegion":"us-east-2","sourceIPAddress":"89.160.20.156","userAgent":"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0","errorMessage":"Failed authentication","requestParameters":null,"responseElements":{"ConsoleLogin":"Failure"},"additionalEventData":{"MobileVersion":"No","LoginTo":"https://console.aws.amazon.com/sns","MFAUsed":"No"},"eventID":"11ea990b-4678-4bcd-8fbe-625EXAMPLE"} diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-console-login-json.log-config.yml b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-console-login-json.log-config.yml deleted file mode 100644 index c39dc38617..0000000000 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-console-login-json.log-config.yml +++ /dev/null @@ -1,2 +0,0 @@ -dynamic_fields: - event.ingested: ".*" diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-console-login-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-console-login-json.log-expected.json index 0d16e8864f..69fec10edc 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-console-login-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-console-login-json.log-expected.json @@ -1,6 +1,31 @@ { "expected": [ { + "source": { + "geo": { + "continent_name": "Europe", + "region_iso_code": "SE-AB", + "city_name": "Tumba", + "country_iso_code": "SE", + "country_name": "Sweden", + "region_name": "Stockholm", + "location": { + "lon": 17.8167, + "lat": 59.2 + } + }, + "as": { + "number": 29518, + "organization": { + "name": "Bredband2 AB" + } + }, + "address": "89.160.20.156", + "ip": "89.160.20.156" + }, + "tags": [ + "preserve_original_event" + ], "cloud": { "region": "us-east-2", "account": { @@ -8,20 +33,18 @@ } }, "@timestamp": "2014-07-16T15:49:27.000Z", + "ecs": { + "version": "1.12.0" + }, "related": { "user": [ "JohnDoe" ] }, - "source": { - "address": "192.0.2.110", - "ip": "192.0.2.110" - }, "event": { - "ingested": "2021-04-23T12:15:54.469997177Z", - "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"AIDACKCEVSQ6C2EXAMPLE\",\"arn\":\"arn:aws:iam::111122223333:user/JohnDoe\",\"accountId\":\"111122223333\",\"userName\":\"JohnDoe\"},\"eventTime\":\"2014-07-16T15:49:27Z\",\"eventSource\":\"signin.amazonaws.com\",\"eventName\":\"ConsoleLogin\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"192.0.2.110\",\"userAgent\":\"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0\",\"requestParameters\":null,\"responseElements\":{\"ConsoleLogin\":\"Success\"},\"additionalEventData\":{\"MobileVersion\":\"No\",\"LoginTo\":\"https://console.aws.amazon.com/s3/\",\"MFAUsed\":\"No\"},\"eventID\":\"3fcfb182-98f8-4744-bd45-10aEXAMPLE\"}", + "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"AIDACKCEVSQ6C2EXAMPLE\",\"arn\":\"arn:aws:iam::111122223333:user/JohnDoe\",\"accountId\":\"111122223333\",\"userName\":\"JohnDoe\"},\"eventTime\":\"2014-07-16T15:49:27Z\",\"eventSource\":\"signin.amazonaws.com\",\"eventName\":\"ConsoleLogin\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"89.160.20.156\",\"userAgent\":\"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0\",\"requestParameters\":null,\"responseElements\":{\"ConsoleLogin\":\"Success\"},\"additionalEventData\":{\"MobileVersion\":\"No\",\"LoginTo\":\"https://console.aws.amazon.com/s3/\",\"MFAUsed\":\"No\"},\"eventID\":\"3fcfb182-98f8-4744-bd45-10aEXAMPLE\"}", "provider": "signin.amazonaws.com", - "created": "2014-07-16T15:49:27.000Z", + "created": "2021-11-11T01:02:03.123456789Z", "kind": "event", "action": "ConsoleLogin", "id": "3fcfb182-98f8-4744-bd45-10aEXAMPLE", @@ -80,6 +103,31 @@ } }, { + "source": { + "geo": { + "continent_name": "Europe", + "region_iso_code": "SE-AB", + "city_name": "Tumba", + "country_iso_code": "SE", + "country_name": "Sweden", + "region_name": "Stockholm", + "location": { + "lon": 17.8167, + "lat": 59.2 + } + }, + "as": { + "number": 29518, + "organization": { + "name": "Bredband2 AB" + } + }, + "address": "89.160.20.156", + "ip": "89.160.20.156" + }, + "tags": [ + "preserve_original_event" + ], "cloud": { "region": "us-east-2", "account": { @@ -87,20 +135,18 @@ } }, "@timestamp": "2014-07-08T17:35:27.000Z", + "ecs": { + "version": "1.12.0" + }, "related": { "user": [ "JaneDoe" ] }, - "source": { - "address": "192.0.2.100", - "ip": "192.0.2.100" - }, "event": { - "ingested": "2021-04-23T12:15:54.470006220Z", - "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"AIDACKCEVSQ6C2EXAMPLE\",\"arn\":\"arn:aws:iam::111122223333:user/JaneDoe\",\"accountId\":\"111122223333\",\"userName\":\"JaneDoe\"},\"eventTime\":\"2014-07-08T17:35:27Z\",\"eventSource\":\"signin.amazonaws.com\",\"eventName\":\"ConsoleLogin\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"192.0.2.100\",\"userAgent\":\"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0\",\"errorMessage\":\"Failed authentication\",\"requestParameters\":null,\"responseElements\":{\"ConsoleLogin\":\"Failure\"},\"additionalEventData\":{\"MobileVersion\":\"No\",\"LoginTo\":\"https://console.aws.amazon.com/sns\",\"MFAUsed\":\"No\"},\"eventID\":\"11ea990b-4678-4bcd-8fbe-625EXAMPLE\"}", + "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"AIDACKCEVSQ6C2EXAMPLE\",\"arn\":\"arn:aws:iam::111122223333:user/JaneDoe\",\"accountId\":\"111122223333\",\"userName\":\"JaneDoe\"},\"eventTime\":\"2014-07-08T17:35:27Z\",\"eventSource\":\"signin.amazonaws.com\",\"eventName\":\"ConsoleLogin\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"89.160.20.156\",\"userAgent\":\"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0\",\"errorMessage\":\"Failed authentication\",\"requestParameters\":null,\"responseElements\":{\"ConsoleLogin\":\"Failure\"},\"additionalEventData\":{\"MobileVersion\":\"No\",\"LoginTo\":\"https://console.aws.amazon.com/sns\",\"MFAUsed\":\"No\"},\"eventID\":\"11ea990b-4678-4bcd-8fbe-625EXAMPLE\"}", "provider": "signin.amazonaws.com", - "created": "2014-07-08T17:35:27.000Z", + "created": "2021-11-11T01:02:03.123456789Z", "kind": "event", "action": "ConsoleLogin", "id": "11ea990b-4678-4bcd-8fbe-625EXAMPLE", @@ -167,15 +213,35 @@ } }, "@timestamp": "2014-07-08T17:35:27.000Z", + "ecs": { + "version": "1.12.0" + }, "source": { - "address": "192.0.2.100", - "ip": "192.0.2.100" + "geo": { + "continent_name": "Europe", + "region_iso_code": "SE-AB", + "city_name": "Tumba", + "country_iso_code": "SE", + "country_name": "Sweden", + "region_name": "Stockholm", + "location": { + "lon": 17.8167, + "lat": 59.2 + } + }, + "as": { + "number": 29518, + "organization": { + "name": "Bredband2 AB" + } + }, + "address": "89.160.20.156", + "ip": "89.160.20.156" }, "event": { - "ingested": "2021-04-23T12:15:54.470010588Z", - "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"AssumedRole\",\"principalId\":\"AROAIDPPEZS35WEXAMPLE:AssumedRoleSessionName\",\"arn\":\"arn:aws:sts::123456789012:assumed-role/RoleToBeAssumed/MySessionName\",\"accountId\":\"123456789012\",\"accessKeyId\":\"AKIAIOSFODNN7EXAMPLE\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"false\",\"creationDate\":\"20131102T010628Z\"},\"sessionIssuer\":{\"type\":\"Role\",\"principalId\":\"AROAIDPPEZS35WEXAMPLE\",\"arn\":\"arn:aws:iam::123456789012:role/RoleToBeAssumed\",\"accountId\":\"123456789012\",\"userName\":\"RoleToBeAssumed\"}}},\"eventTime\":\"2014-07-08T17:35:27Z\",\"eventSource\":\"signin.amazonaws.com\",\"eventName\":\"ConsoleLogin\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"192.0.2.100\",\"userAgent\":\"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0\",\"errorMessage\":\"Failed authentication\",\"requestParameters\":null,\"responseElements\":{\"ConsoleLogin\":\"Failure\"},\"additionalEventData\":{\"MobileVersion\":\"No\",\"LoginTo\":\"https://console.aws.amazon.com/sns\",\"MFAUsed\":\"No\"},\"eventID\":\"11ea990b-4678-4bcd-8fbe-625EXAMPLE\"}", + "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"AssumedRole\",\"principalId\":\"AROAIDPPEZS35WEXAMPLE:AssumedRoleSessionName\",\"arn\":\"arn:aws:sts::123456789012:assumed-role/RoleToBeAssumed/MySessionName\",\"accountId\":\"123456789012\",\"accessKeyId\":\"AKIAIOSFODNN7EXAMPLE\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"false\",\"creationDate\":\"20131102T010628Z\"},\"sessionIssuer\":{\"type\":\"Role\",\"principalId\":\"AROAIDPPEZS35WEXAMPLE\",\"arn\":\"arn:aws:iam::123456789012:role/RoleToBeAssumed\",\"accountId\":\"123456789012\",\"userName\":\"RoleToBeAssumed\"}}},\"eventTime\":\"2014-07-08T17:35:27Z\",\"eventSource\":\"signin.amazonaws.com\",\"eventName\":\"ConsoleLogin\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"89.160.20.156\",\"userAgent\":\"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0\",\"errorMessage\":\"Failed authentication\",\"requestParameters\":null,\"responseElements\":{\"ConsoleLogin\":\"Failure\"},\"additionalEventData\":{\"MobileVersion\":\"No\",\"LoginTo\":\"https://console.aws.amazon.com/sns\",\"MFAUsed\":\"No\"},\"eventID\":\"11ea990b-4678-4bcd-8fbe-625EXAMPLE\"}", "provider": "signin.amazonaws.com", - "created": "2014-07-08T17:35:27.000Z", + "created": "2021-11-11T01:02:03.123456789Z", "kind": "event", "action": "ConsoleLogin", "id": "11ea990b-4678-4bcd-8fbe-625EXAMPLE", @@ -242,7 +308,10 @@ "name": "Other" }, "version": "24.0." - } + }, + "tags": [ + "preserve_original_event" + ] } ] } \ No newline at end of file diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-access-key-json.log-config.yml b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-access-key-json.log-config.yml deleted file mode 100644 index c39dc38617..0000000000 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-access-key-json.log-config.yml +++ /dev/null @@ -1,2 +0,0 @@ -dynamic_fields: - event.ingested: ".*" diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-access-key-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-access-key-json.log-expected.json index 4b5721cf4c..fe958a8e1b 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-access-key-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-access-key-json.log-expected.json @@ -1,6 +1,13 @@ { "expected": [ { + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "tags": [ + "preserve_original_event" + ], "cloud": { "region": "us-east-1", "account": { @@ -8,21 +15,19 @@ } }, "@timestamp": "2020-01-08T20:43:06.000Z", + "ecs": { + "version": "1.12.0" + }, "related": { "user": [ "Alice", "Bob" ] }, - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, "event": { - "ingested": "2021-04-23T12:15:54.575565508Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-08T15:12:16Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-08T20:43:06Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"CreateAccessKey\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"userName\":\"Bob\"},\"responseElements\":{\"accessKey\":{\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"status\":\"Active\",\"userName\":\"Bob\",\"createDate\":\"Jan 8, 2020 8:43:06 PM\"}},\"requestID\":\"EXAMPLE-823a-48dc-8fa9-EXAMPLE\",\"eventID\":\"EXAMPLE-3cab-40f8-938b-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", - "created": "2020-01-08T20:43:06.000Z", + "created": "2021-11-11T01:02:03.123456789Z", "kind": "event", "action": "CreateAccessKey", "id": "EXAMPLE-3cab-40f8-938b-EXAMPLE", diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-group-json.log-config.yml b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-group-json.log-config.yml deleted file mode 100644 index c39dc38617..0000000000 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-group-json.log-config.yml +++ /dev/null @@ -1,2 +0,0 @@ -dynamic_fields: - event.ingested: ".*" diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-group-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-group-json.log-expected.json index a7d9d907fb..e8a8378aac 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-group-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-group-json.log-expected.json @@ -1,6 +1,13 @@ { "expected": [ { + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "tags": [ + "preserve_original_event" + ], "cloud": { "region": "us-east-1", "account": { @@ -8,20 +15,18 @@ } }, "@timestamp": "2020-01-09T01:48:44.000Z", + "ecs": { + "version": "1.12.0" + }, "related": { "user": [ "Alice" ] }, - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, "event": { - "ingested": "2021-04-23T12:15:54.610901167Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"0123456789012\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-08T15:12:16Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-09T01:48:44Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"CreateGroup\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"groupName\":\"TEST-GROUP\"},\"responseElements\":{\"group\":{\"createDate\":\"Jan 9, 2020 1:48:44 AM\",\"path\":\"/\",\"arn\":\"arn:aws:iam::0123456789012:group/TEST-GROUP\",\"groupName\":\"TEST-GROUP\",\"groupId\":\"EXAMPLE_ID\"}},\"requestID\":\"EXAMPLE-769d-4a61-b731-EXAMPLE\",\"eventID\":\"EXAMPLE-37ec-425a-a7ef-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", - "created": "2020-01-09T01:48:44.000Z", + "created": "2021-11-11T01:02:03.123456789Z", "kind": "event", "action": "CreateGroup", "id": "EXAMPLE-37ec-425a-a7ef-EXAMPLE", @@ -84,6 +89,13 @@ } }, { + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "tags": [ + "preserve_original_event" + ], "cloud": { "region": "us-east-1", "account": { @@ -91,20 +103,18 @@ } }, "@timestamp": "2020-01-09T02:22:03.000Z", + "ecs": { + "version": "1.12.0" + }, "related": { "user": [ "Alice" ] }, - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, "event": { - "ingested": "2021-04-23T12:15:54.610909336Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"0123456789012\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\"},\"eventTime\":\"2020-01-09T02:22:03Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"CreateGroup\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46\",\"errorCode\":\"EntityAlreadyExistsException\",\"errorMessage\":\"Group with name TEST-GROUP already exists.\",\"requestParameters\":{\"groupName\":\"TEST-GROUP\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-c8ae-44dc-8114-EXAMPLE\",\"eventID\":\"EXAMPLE-09c6-4745-af70-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", - "created": "2020-01-09T02:22:03.000Z", + "created": "2021-11-11T01:02:03.123456789Z", "kind": "event", "action": "CreateGroup", "id": "EXAMPLE-09c6-4745-af70-EXAMPLE", diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-key-pair-json.log b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-key-pair-json.log index 5b9c40ad40..81f2d01071 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-key-pair-json.log +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-key-pair-json.log @@ -1 +1 @@ -{"eventVersion":"1.0","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice","sessionContext":{"attributes":{"mfaAuthenticated":"false","creationDate":"2014-03-06T15:15:06Z"}}},"eventTime":"2014-03-06T17:10:34Z","eventSource":"ec2.amazonaws.com","eventName":"CreateKeyPair","awsRegion":"us-east-2","sourceIPAddress":"72.21.198.64","userAgent":"EC2ConsoleBackend, aws-sdk-java/Linux/x.xx.fleetxen Java_HotSpot(TM)_64-Bit_Server_VM/xx","requestParameters":{"keyName":"mykeypair"},"responseElements":{"keyName":"mykeypair","keyFingerprint":"30:1d:46:d0:5b:ad:7e:1b:b6:70:62:8b:ff:38:b5:e9:ab:5d:b8:21","keyMaterial":""}} +{"eventVersion":"1.0","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice","sessionContext":{"attributes":{"mfaAuthenticated":"false","creationDate":"2014-03-06T15:15:06Z"}}},"eventTime":"2014-03-06T17:10:34Z","eventSource":"ec2.amazonaws.com","eventName":"CreateKeyPair","awsRegion":"us-east-2","sourceIPAddress":"89.160.20.156","userAgent":"EC2ConsoleBackend, aws-sdk-java/Linux/x.xx.fleetxen Java_HotSpot(TM)_64-Bit_Server_VM/xx","requestParameters":{"keyName":"mykeypair"},"responseElements":{"keyName":"mykeypair","keyFingerprint":"30:1d:46:d0:5b:ad:7e:1b:b6:70:62:8b:ff:38:b5:e9:ab:5d:b8:21","keyMaterial":""}} diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-key-pair-json.log-config.yml b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-key-pair-json.log-config.yml deleted file mode 100644 index c39dc38617..0000000000 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-key-pair-json.log-config.yml +++ /dev/null @@ -1,2 +0,0 @@ -dynamic_fields: - event.ingested: ".*" diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-key-pair-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-key-pair-json.log-expected.json index a983c0786d..00bd7e5645 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-key-pair-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-key-pair-json.log-expected.json @@ -1,6 +1,31 @@ { "expected": [ { + "source": { + "geo": { + "continent_name": "Europe", + "region_iso_code": "SE-AB", + "city_name": "Tumba", + "country_iso_code": "SE", + "country_name": "Sweden", + "region_name": "Stockholm", + "location": { + "lon": 17.8167, + "lat": 59.2 + } + }, + "as": { + "number": 29518, + "organization": { + "name": "Bredband2 AB" + } + }, + "address": "89.160.20.156", + "ip": "89.160.20.156" + }, + "tags": [ + "preserve_original_event" + ], "cloud": { "region": "us-east-2", "account": { @@ -8,38 +33,18 @@ } }, "@timestamp": "2014-03-06T17:10:34.000Z", + "ecs": { + "version": "1.12.0" + }, "related": { "user": [ "Alice" ] }, - "source": { - "geo": { - "continent_name": "North America", - "region_iso_code": "US-VA", - "city_name": "Ashburn", - "country_iso_code": "US", - "country_name": "United States", - "region_name": "Virginia", - "location": { - "lon": -77.4728, - "lat": 39.0481 - } - }, - "as": { - "number": 16509, - "organization": { - "name": "Amazon.com, Inc." - } - }, - "address": "72.21.198.64", - "ip": "72.21.198.64" - }, "event": { - "ingested": "2021-04-23T12:15:54.676175189Z", - "original": "{\"eventVersion\":\"1.0\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EX_PRINCIPAL_ID\",\"arn\":\"arn:aws:iam::123456789012:user/Alice\",\"accountId\":\"123456789012\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"false\",\"creationDate\":\"2014-03-06T15:15:06Z\"}}},\"eventTime\":\"2014-03-06T17:10:34Z\",\"eventSource\":\"ec2.amazonaws.com\",\"eventName\":\"CreateKeyPair\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"72.21.198.64\",\"userAgent\":\"EC2ConsoleBackend, aws-sdk-java/Linux/x.xx.fleetxen Java_HotSpot(TM)_64-Bit_Server_VM/xx\",\"requestParameters\":{\"keyName\":\"mykeypair\"},\"responseElements\":{\"keyName\":\"mykeypair\",\"keyFingerprint\":\"30:1d:46:d0:5b:ad:7e:1b:b6:70:62:8b:ff:38:b5:e9:ab:5d:b8:21\",\"keyMaterial\":\"\u003csensitiveDataRemoved\u003e\"}}", + "original": "{\"eventVersion\":\"1.0\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EX_PRINCIPAL_ID\",\"arn\":\"arn:aws:iam::123456789012:user/Alice\",\"accountId\":\"123456789012\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"false\",\"creationDate\":\"2014-03-06T15:15:06Z\"}}},\"eventTime\":\"2014-03-06T17:10:34Z\",\"eventSource\":\"ec2.amazonaws.com\",\"eventName\":\"CreateKeyPair\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"89.160.20.156\",\"userAgent\":\"EC2ConsoleBackend, aws-sdk-java/Linux/x.xx.fleetxen Java_HotSpot(TM)_64-Bit_Server_VM/xx\",\"requestParameters\":{\"keyName\":\"mykeypair\"},\"responseElements\":{\"keyName\":\"mykeypair\",\"keyFingerprint\":\"30:1d:46:d0:5b:ad:7e:1b:b6:70:62:8b:ff:38:b5:e9:ab:5d:b8:21\",\"keyMaterial\":\"\u003csensitiveDataRemoved\u003e\"}}", "provider": "ec2.amazonaws.com", - "created": "2014-03-06T17:10:34.000Z", + "created": "2021-11-11T01:02:03.123456789Z", "kind": "event", "action": "CreateKeyPair", "type": [ diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-trail-json.log-config.yml b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-trail-json.log-config.yml deleted file mode 100644 index c39dc38617..0000000000 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-trail-json.log-config.yml +++ /dev/null @@ -1,2 +0,0 @@ -dynamic_fields: - event.ingested: ".*" diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-trail-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-trail-json.log-expected.json index d1a2edbbfa..1839c2927b 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-trail-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-trail-json.log-expected.json @@ -1,6 +1,13 @@ { "expected": [ { + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "tags": [ + "preserve_original_event" + ], "cloud": { "region": "us-west-2", "account": { @@ -8,20 +15,18 @@ } }, "@timestamp": "2020-01-08T15:30:25.000Z", + "ecs": { + "version": "1.12.0" + }, "related": { "user": [ "Alice" ] }, - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, "event": { - "ingested": "2021-04-23T12:15:54.718155521Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"sessionIssuer\":{},\"webIdFederationData\":{},\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-08T15:12:16Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-08T15:30:25Z\",\"eventSource\":\"cloudtrail.amazonaws.com\",\"eventName\":\"CreateTrail\",\"awsRegion\":\"us-west-2\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"name\":\"TEST-trail\",\"s3BucketName\":\"TEST-cloudtrail-bucket\",\"includeGlobalServiceEvents\":true,\"isMultiRegionTrail\":true,\"enableLogFileValidation\":true,\"kmsKeyId\":\"\",\"isOrganizationTrail\":false},\"responseElements\":{\"name\":\"TEST-trail\",\"s3BucketName\":\"TEST-cloudtrail-bucket\",\"includeGlobalServiceEvents\":true,\"isMultiRegionTrail\":true,\"trailARN\":\"arn:aws:cloudtrail:us-west-2:0123456789012:trail/TEST-trail\",\"logFileValidationEnabled\":true,\"isOrganizationTrail\":false},\"requestID\":\"EXAMPLE-5149-4cf2-be99-EXAMPLE\",\"eventID\":\"EXAMPLE-d04b-4eff-833a-EXAMPLE\",\"readOnly\":false,\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "cloudtrail.amazonaws.com", - "created": "2020-01-08T15:30:25.000Z", + "created": "2021-11-11T01:02:03.123456789Z", "kind": "event", "action": "CreateTrail", "id": "EXAMPLE-d04b-4eff-833a-EXAMPLE", diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-user-json.log-config.yml b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-user-json.log-config.yml deleted file mode 100644 index c39dc38617..0000000000 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-user-json.log-config.yml +++ /dev/null @@ -1,2 +0,0 @@ -dynamic_fields: - event.ingested: ".*" diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-user-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-user-json.log-expected.json index 9faea88ba2..5fc1e895b7 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-user-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-user-json.log-expected.json @@ -1,6 +1,13 @@ { "expected": [ { + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "tags": [ + "preserve_original_event" + ], "cloud": { "region": "us-east-2", "account": { @@ -8,21 +15,19 @@ } }, "@timestamp": "2014-03-24T21:11:59.000Z", + "ecs": { + "version": "1.12.0" + }, "related": { "user": [ "Alice", "Bob" ] }, - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, "event": { - "ingested": "2021-04-23T12:15:54.752417207Z", "original": "{\"eventVersion\":\"1.0\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EX_PRINCIPAL_ID\",\"arn\":\"arn:aws:iam::123456789012:user/Alice\",\"accountId\":\"123456789012\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Alice\"},\"eventTime\":\"2014-03-24T21:11:59Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"CreateUser\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"aws-cli/1.3.2 Python/2.7.5 Windows/7\",\"requestParameters\":{\"userName\":\"Bob\"},\"responseElements\":{\"user\":{\"createDate\":\"Mar 24, 2014 9:11:59 PM\",\"userName\":\"Bob\",\"arn\":\"arn:aws:iam::123456789012:user/Bob\",\"path\":\"/\",\"userId\":\"EXAMPLEUSERID\"}}}", "provider": "iam.amazonaws.com", - "created": "2014-03-24T21:11:59.000Z", + "created": "2021-11-11T01:02:03.123456789Z", "kind": "event", "action": "CreateUser", "type": [ diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-virtual-mfa-device-json.log-config.yml b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-virtual-mfa-device-json.log-config.yml deleted file mode 100644 index c39dc38617..0000000000 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-virtual-mfa-device-json.log-config.yml +++ /dev/null @@ -1,2 +0,0 @@ -dynamic_fields: - event.ingested: ".*" diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-virtual-mfa-device-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-virtual-mfa-device-json.log-expected.json index e01fbdd67b..4a9c4f4240 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-virtual-mfa-device-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-virtual-mfa-device-json.log-expected.json @@ -1,6 +1,13 @@ { "expected": [ { + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "tags": [ + "preserve_original_event" + ], "cloud": { "region": "us-east-1", "account": { @@ -8,20 +15,18 @@ } }, "@timestamp": "2019-11-27T15:10:15.000Z", + "ecs": { + "version": "1.12.0" + }, "related": { "user": [ "Alice" ] }, - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, "event": { - "ingested": "2021-04-23T12:15:54.784993236Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"false\",\"creationDate\":\"2019-11-27T15:07:22Z\"}}},\"eventTime\":\"2019-11-27T15:10:15Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"CreateVirtualMFADevice\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"console.amazonaws.com\",\"requestParameters\":{\"virtualMFADeviceName\":\"Alice\",\"path\":\"/\"},\"responseElements\":{\"virtualMFADevice\":{\"serialNumber\":\"arn:aws:iam::0123456789012:mfa/Alice\"}},\"requestID\":\"EXAMPLE-303b-4b0e-a8c7-EXAMPLE\",\"eventID\":\"EXAMPLE-351c-472a-b089-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", - "created": "2019-11-27T15:10:15.000Z", + "created": "2021-11-11T01:02:03.123456789Z", "kind": "event", "action": "CreateVirtualMFADevice", "id": "EXAMPLE-351c-472a-b089-EXAMPLE", diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-deactivate-mfa-device-json.log-config.yml b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-deactivate-mfa-device-json.log-config.yml deleted file mode 100644 index c39dc38617..0000000000 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-deactivate-mfa-device-json.log-config.yml +++ /dev/null @@ -1,2 +0,0 @@ -dynamic_fields: - event.ingested: ".*" diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-deactivate-mfa-device-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-deactivate-mfa-device-json.log-expected.json index c9f1ee9e85..764cc3d905 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-deactivate-mfa-device-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-deactivate-mfa-device-json.log-expected.json @@ -1,6 +1,13 @@ { "expected": [ { + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "tags": [ + "preserve_original_event" + ], "cloud": { "region": "us-east-1", "account": { @@ -8,20 +15,18 @@ } }, "@timestamp": "2020-01-10T00:34:02.000Z", + "ecs": { + "version": "1.12.0" + }, "related": { "user": [ "Alice" ] }, - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, "event": { - "ingested": "2021-04-23T12:15:54.818318682Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_ID\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-09T16:36:17Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-10T00:34:02Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"DeactivateMFADevice\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"userName\":\"Alice\",\"serialNumber\":\"arn:aws:iam::0123456789012:mfa/Alice\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-801a-4624-8fa0-EXAMPLE\",\"eventID\":\"EXAMPLE-1889-416b-ace9-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", - "created": "2020-01-10T00:34:02.000Z", + "created": "2021-11-11T01:02:03.123456789Z", "kind": "event", "action": "DeactivateMFADevice", "id": "EXAMPLE-1889-416b-ace9-EXAMPLE", diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-access-key-json.log-config.yml b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-access-key-json.log-config.yml deleted file mode 100644 index c39dc38617..0000000000 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-access-key-json.log-config.yml +++ /dev/null @@ -1,2 +0,0 @@ -dynamic_fields: - event.ingested: ".*" diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-access-key-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-access-key-json.log-expected.json index 55f9599c3b..885f9b97c9 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-access-key-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-access-key-json.log-expected.json @@ -1,6 +1,13 @@ { "expected": [ { + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "tags": [ + "preserve_original_event" + ], "cloud": { "region": "us-east-1", "account": { @@ -8,21 +15,19 @@ } }, "@timestamp": "2020-01-08T19:09:36.000Z", + "ecs": { + "version": "1.12.0" + }, "related": { "user": [ "Alice", "Bob" ] }, - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, "event": { - "ingested": "2021-04-23T12:15:54.852318084Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_ID\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-08T15:12:16Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-08T19:09:36Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"DeleteAccessKey\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"userName\":\"Bob\",\"accessKeyId\":\"EXAMPLE_ID\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-3bea-41fa-a0b4-EXAMPLE\",\"eventID\":\"EXAMPLE-0698-46bd-998d-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", - "created": "2020-01-08T19:09:36.000Z", + "created": "2021-11-11T01:02:03.123456789Z", "kind": "event", "action": "DeleteAccessKey", "id": "EXAMPLE-0698-46bd-998d-EXAMPLE", diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-bucket-json.log b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-bucket-json.log index 913b109d7c..ab5c34153a 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-bucket-json.log +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-bucket-json.log @@ -1 +1 @@ -{"eventVersion":"1.04","userIdentity":{"type":"AssumedRole","principalId":"AIDAQRSTUVWXYZEXAMPLE:devdsk","arn":"arn:aws:sts::777788889999:assumed-role/AssumeNothing/devdsk","accountId":"777788889999","accessKeyId":"AKIAQRSTUVWXYZEXAMPLE","sessionContext":{"attributes":{"mfaAuthenticated":"false","creationDate":"2016-11-14T17:25:26Z"},"sessionIssuer":{"type":"Role","principalId":"AIDAQRSTUVWXYZEXAMPLE","arn":"arn:aws:iam::777788889999:role/AssumeNothing","accountId":"777788889999","userName":"AssumeNothing"}}},"eventTime":"2016-11-14T17:25:45Z","eventSource":"s3.amazonaws.com","eventName":"DeleteBucket","awsRegion":"us-east-2","sourceIPAddress":"192.0.2.1","userAgent":"[aws-cli/1.11.10 Python/2.7.8 Linux/3.2.45-0.6.wd.865.49.315.metal1.x86_64 botocore/1.4.67]","requestParameters":{"bucketName":"my-test-bucket-cross-account"},"responseElements":null,"requestID":"EXAMPLE463D56D4C","eventID":"dEXAMPLE-265a-41e0-9352-4401bEXAMPLE","eventType":"AwsApiCall","recipientAccountId":"777788889999"} +{"eventVersion":"1.04","userIdentity":{"type":"AssumedRole","principalId":"AIDAQRSTUVWXYZEXAMPLE:devdsk","arn":"arn:aws:sts::777788889999:assumed-role/AssumeNothing/devdsk","accountId":"777788889999","accessKeyId":"AKIAQRSTUVWXYZEXAMPLE","sessionContext":{"attributes":{"mfaAuthenticated":"false","creationDate":"2016-11-14T17:25:26Z"},"sessionIssuer":{"type":"Role","principalId":"AIDAQRSTUVWXYZEXAMPLE","arn":"arn:aws:iam::777788889999:role/AssumeNothing","accountId":"777788889999","userName":"AssumeNothing"}}},"eventTime":"2016-11-14T17:25:45Z","eventSource":"s3.amazonaws.com","eventName":"DeleteBucket","awsRegion":"us-east-2","sourceIPAddress":"89.160.20.156","userAgent":"[aws-cli/1.11.10 Python/2.7.8 Linux/3.2.45-0.6.wd.865.49.315.metal1.x86_64 botocore/1.4.67]","requestParameters":{"bucketName":"my-test-bucket-cross-account"},"responseElements":null,"requestID":"EXAMPLE463D56D4C","eventID":"dEXAMPLE-265a-41e0-9352-4401bEXAMPLE","eventType":"AwsApiCall","recipientAccountId":"777788889999"} diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-bucket-json.log-config.yml b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-bucket-json.log-config.yml deleted file mode 100644 index c39dc38617..0000000000 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-bucket-json.log-config.yml +++ /dev/null @@ -1,2 +0,0 @@ -dynamic_fields: - event.ingested: ".*" diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-bucket-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-bucket-json.log-expected.json index e5328c7f5d..4ed161acea 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-bucket-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-bucket-json.log-expected.json @@ -8,15 +8,35 @@ } }, "@timestamp": "2016-11-14T17:25:45.000Z", + "ecs": { + "version": "1.12.0" + }, "source": { - "address": "192.0.2.1", - "ip": "192.0.2.1" + "geo": { + "continent_name": "Europe", + "region_iso_code": "SE-AB", + "city_name": "Tumba", + "country_iso_code": "SE", + "country_name": "Sweden", + "region_name": "Stockholm", + "location": { + "lon": 17.8167, + "lat": 59.2 + } + }, + "as": { + "number": 29518, + "organization": { + "name": "Bredband2 AB" + } + }, + "address": "89.160.20.156", + "ip": "89.160.20.156" }, "event": { - "ingested": "2021-04-23T12:15:54.886661019Z", - "original": "{\"eventVersion\":\"1.04\",\"userIdentity\":{\"type\":\"AssumedRole\",\"principalId\":\"AIDAQRSTUVWXYZEXAMPLE:devdsk\",\"arn\":\"arn:aws:sts::777788889999:assumed-role/AssumeNothing/devdsk\",\"accountId\":\"777788889999\",\"accessKeyId\":\"AKIAQRSTUVWXYZEXAMPLE\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"false\",\"creationDate\":\"2016-11-14T17:25:26Z\"},\"sessionIssuer\":{\"type\":\"Role\",\"principalId\":\"AIDAQRSTUVWXYZEXAMPLE\",\"arn\":\"arn:aws:iam::777788889999:role/AssumeNothing\",\"accountId\":\"777788889999\",\"userName\":\"AssumeNothing\"}}},\"eventTime\":\"2016-11-14T17:25:45Z\",\"eventSource\":\"s3.amazonaws.com\",\"eventName\":\"DeleteBucket\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"192.0.2.1\",\"userAgent\":\"[aws-cli/1.11.10 Python/2.7.8 Linux/3.2.45-0.6.wd.865.49.315.metal1.x86_64 botocore/1.4.67]\",\"requestParameters\":{\"bucketName\":\"my-test-bucket-cross-account\"},\"responseElements\":null,\"requestID\":\"EXAMPLE463D56D4C\",\"eventID\":\"dEXAMPLE-265a-41e0-9352-4401bEXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"777788889999\"}", + "original": "{\"eventVersion\":\"1.04\",\"userIdentity\":{\"type\":\"AssumedRole\",\"principalId\":\"AIDAQRSTUVWXYZEXAMPLE:devdsk\",\"arn\":\"arn:aws:sts::777788889999:assumed-role/AssumeNothing/devdsk\",\"accountId\":\"777788889999\",\"accessKeyId\":\"AKIAQRSTUVWXYZEXAMPLE\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"false\",\"creationDate\":\"2016-11-14T17:25:26Z\"},\"sessionIssuer\":{\"type\":\"Role\",\"principalId\":\"AIDAQRSTUVWXYZEXAMPLE\",\"arn\":\"arn:aws:iam::777788889999:role/AssumeNothing\",\"accountId\":\"777788889999\",\"userName\":\"AssumeNothing\"}}},\"eventTime\":\"2016-11-14T17:25:45Z\",\"eventSource\":\"s3.amazonaws.com\",\"eventName\":\"DeleteBucket\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"89.160.20.156\",\"userAgent\":\"[aws-cli/1.11.10 Python/2.7.8 Linux/3.2.45-0.6.wd.865.49.315.metal1.x86_64 botocore/1.4.67]\",\"requestParameters\":{\"bucketName\":\"my-test-bucket-cross-account\"},\"responseElements\":null,\"requestID\":\"EXAMPLE463D56D4C\",\"eventID\":\"dEXAMPLE-265a-41e0-9352-4401bEXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"777788889999\"}", "provider": "s3.amazonaws.com", - "created": "2016-11-14T17:25:45.000Z", + "created": "2021-11-11T01:02:03.123456789Z", "kind": "event", "action": "DeleteBucket", "id": "dEXAMPLE-265a-41e0-9352-4401bEXAMPLE", @@ -72,7 +92,10 @@ "name": "Spider" }, "version": "1.11.10" - } + }, + "tags": [ + "preserve_original_event" + ] } ] } \ No newline at end of file diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-group-json.log-config.yml b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-group-json.log-config.yml deleted file mode 100644 index c39dc38617..0000000000 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-group-json.log-config.yml +++ /dev/null @@ -1,2 +0,0 @@ -dynamic_fields: - event.ingested: ".*" diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-group-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-group-json.log-expected.json index 3b62aa1a31..4495b77d45 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-group-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-group-json.log-expected.json @@ -1,6 +1,13 @@ { "expected": [ { + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "tags": [ + "preserve_original_event" + ], "cloud": { "region": "us-east-1", "account": { @@ -8,20 +15,18 @@ } }, "@timestamp": "2020-01-09T02:25:44.000Z", + "ecs": { + "version": "1.12.0" + }, "related": { "user": [ "Alice" ] }, - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, "event": { - "ingested": "2021-04-23T12:15:54.924240276Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"0123456789012\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-08T15:12:16Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-09T02:25:44Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"DeleteGroup\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"groupName\":\"TEST-GROUP\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-66cb-4775-a203-EXAMPLE\",\"eventID\":\"EXAMPLE-cbc2-4cc3-8bbc-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", - "created": "2020-01-09T02:25:44.000Z", + "created": "2021-11-11T01:02:03.123456789Z", "kind": "event", "action": "DeleteGroup", "id": "EXAMPLE-cbc2-4cc3-8bbc-EXAMPLE", @@ -73,6 +78,13 @@ } }, { + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "tags": [ + "preserve_original_event" + ], "cloud": { "region": "us-east-1", "account": { @@ -80,20 +92,18 @@ } }, "@timestamp": "2020-01-09T02:25:11.000Z", + "ecs": { + "version": "1.12.0" + }, "related": { "user": [ "Alice" ] }, - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, "event": { - "ingested": "2021-04-23T12:15:54.924251898Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_PRINCIPLE\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Alice\"},\"eventTime\":\"2020-01-09T02:25:11Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"DeleteGroup\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46\",\"errorCode\":\"DeleteConflictException\",\"errorMessage\":\"Cannot delete entity, must detach all policies first.\",\"requestParameters\":{\"groupName\":\"TEST-GROUP\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-2a3c-4a94-b24f-EXAMPLE\",\"eventID\":\"EXAMPLE-5aa2-4b5f-a52a-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", - "created": "2020-01-09T02:25:11.000Z", + "created": "2021-11-11T01:02:03.123456789Z", "kind": "event", "action": "DeleteGroup", "id": "EXAMPLE-5aa2-4b5f-a52a-EXAMPLE", diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-ssh-public-key-json.log-config.yml b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-ssh-public-key-json.log-config.yml deleted file mode 100644 index c39dc38617..0000000000 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-ssh-public-key-json.log-config.yml +++ /dev/null @@ -1,2 +0,0 @@ -dynamic_fields: - event.ingested: ".*" diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-ssh-public-key-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-ssh-public-key-json.log-expected.json index e775452704..6ffa98520d 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-ssh-public-key-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-ssh-public-key-json.log-expected.json @@ -1,6 +1,13 @@ { "expected": [ { + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "tags": [ + "preserve_original_event" + ], "cloud": { "region": "us-east-1", "account": { @@ -8,21 +15,19 @@ } }, "@timestamp": "2020-01-10T16:07:08.000Z", + "ecs": { + "version": "1.12.0" + }, "related": { "user": [ "Alice", "Bob" ] }, - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, "event": { - "ingested": "2021-04-23T12:15:54.987346466Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-10T14:38:30Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-10T16:07:08Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"DeleteSSHPublicKey\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"sSHPublicKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Bob\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-7b34-44ae-a22f-EXAMPLE\",\"eventID\":\"EXAMPLE-72ff-4d4f-9a8d-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", - "created": "2020-01-10T16:07:08.000Z", + "created": "2021-11-11T01:02:03.123456789Z", "kind": "event", "action": "DeleteSSHPublicKey", "id": "EXAMPLE-72ff-4d4f-9a8d-EXAMPLE", diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-trail-json.log-config.yml b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-trail-json.log-config.yml deleted file mode 100644 index c39dc38617..0000000000 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-trail-json.log-config.yml +++ /dev/null @@ -1,2 +0,0 @@ -dynamic_fields: - event.ingested: ".*" diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-trail-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-trail-json.log-expected.json index e42dafd2d6..2dd33e9881 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-trail-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-trail-json.log-expected.json @@ -1,6 +1,13 @@ { "expected": [ { + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "tags": [ + "preserve_original_event" + ], "cloud": { "region": "us-west-2", "account": { @@ -8,20 +15,18 @@ } }, "@timestamp": "2020-01-09T20:09:51.000Z", + "ecs": { + "version": "1.12.0" + }, "related": { "user": [ "Alice" ] }, - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, "event": { - "ingested": "2021-04-23T12:15:55.024826729Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\"},\"eventTime\":\"2020-01-09T20:09:51Z\",\"eventSource\":\"cloudtrail.amazonaws.com\",\"eventName\":\"DeleteTrail\",\"awsRegion\":\"us-west-2\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46\",\"requestParameters\":{\"name\":\"arn:aws:cloudtrail:us-west-2:0123456789012:trail/test-trail\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-d44f-4a2a-966f-EXAMPLE\",\"eventID\":\"EXAMPLE-3f9d-4634-8ff1-EXAMPLE\",\"readOnly\":false,\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "cloudtrail.amazonaws.com", - "created": "2020-01-09T20:09:51.000Z", + "created": "2021-11-11T01:02:03.123456789Z", "kind": "event", "action": "DeleteTrail", "id": "EXAMPLE-3f9d-4634-8ff1-EXAMPLE", diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-user-json.log-config.yml b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-user-json.log-config.yml deleted file mode 100644 index c39dc38617..0000000000 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-user-json.log-config.yml +++ /dev/null @@ -1,2 +0,0 @@ -dynamic_fields: - event.ingested: ".*" diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-user-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-user-json.log-expected.json index 52bafe68e4..6bb1d94f1e 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-user-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-user-json.log-expected.json @@ -1,6 +1,13 @@ { "expected": [ { + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "tags": [ + "preserve_original_event" + ], "cloud": { "region": "us-east-1", "account": { @@ -8,21 +15,19 @@ } }, "@timestamp": "2020-01-03T15:50:52.000Z", + "ecs": { + "version": "1.12.0" + }, "related": { "user": [ "Alice", "Bob" ] }, - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, "event": { - "ingested": "2021-04-23T12:15:55.057325041Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EX_PRINCIPAL_ID\",\"arn\":\"arn:aws:iam::123456789012:user/Alice\",\"accountId\":\"123456789012\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-03T15:26:38Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-03T15:50:52Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"DeleteUser\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"userName\":\"Bob\"},\"responseElements\":null,\"requestID\":\"0e794d53-cdb5-4f7d-b7db-5EXAMPLE\",\"eventID\":\"b89eb34b-8fcb-4cba-8439-d4EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"123456789012\"}", "provider": "iam.amazonaws.com", - "created": "2020-01-03T15:50:52.000Z", + "created": "2021-11-11T01:02:03.123456789Z", "kind": "event", "action": "DeleteUser", "id": "b89eb34b-8fcb-4cba-8439-d4EXAMPLE", diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-virtual-mfa-device-json.log-config.yml b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-virtual-mfa-device-json.log-config.yml deleted file mode 100644 index c39dc38617..0000000000 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-virtual-mfa-device-json.log-config.yml +++ /dev/null @@ -1,2 +0,0 @@ -dynamic_fields: - event.ingested: ".*" diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-virtual-mfa-device-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-virtual-mfa-device-json.log-expected.json index c8cb6d6742..a482b81222 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-virtual-mfa-device-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-virtual-mfa-device-json.log-expected.json @@ -1,6 +1,13 @@ { "expected": [ { + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "tags": [ + "preserve_original_event" + ], "cloud": { "region": "us-east-1", "account": { @@ -8,20 +15,18 @@ } }, "@timestamp": "2020-01-10T00:34:02.000Z", + "ecs": { + "version": "1.12.0" + }, "related": { "user": [ "Alice" ] }, - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, "event": { - "ingested": "2021-04-23T12:15:55.091526013Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-09T16:36:17Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-10T00:34:02Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"DeleteVirtualMFADevice\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"serialNumber\":\"arn:aws:iam::0123456789012:mfa/Alice\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-af91-4d1a-aaf2-EXAMPLE\",\"eventID\":\"EXAMPLE-f8e6-4d5f-8525-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", - "created": "2020-01-10T00:34:02.000Z", + "created": "2021-11-11T01:02:03.123456789Z", "kind": "event", "action": "DeleteVirtualMFADevice", "id": "EXAMPLE-f8e6-4d5f-8525-EXAMPLE", diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-enable-mfa-device-json.log-config.yml b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-enable-mfa-device-json.log-config.yml deleted file mode 100644 index c39dc38617..0000000000 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-enable-mfa-device-json.log-config.yml +++ /dev/null @@ -1,2 +0,0 @@ -dynamic_fields: - event.ingested: ".*" diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-enable-mfa-device-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-enable-mfa-device-json.log-expected.json index bff8fbb12b..5a66ad5660 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-enable-mfa-device-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-enable-mfa-device-json.log-expected.json @@ -1,6 +1,13 @@ { "expected": [ { + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "tags": [ + "preserve_original_event" + ], "cloud": { "region": "us-east-1", "account": { @@ -8,21 +15,19 @@ } }, "@timestamp": "2019-11-27T15:11:09.000Z", + "ecs": { + "version": "1.12.0" + }, "related": { "user": [ "Alice", "Bob" ] }, - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, "event": { - "ingested": "2021-04-23T12:15:55.125391948Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"false\",\"creationDate\":\"2019-11-27T15:07:22Z\"}}},\"eventTime\":\"2019-11-27T15:11:09Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"EnableMFADevice\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"console.amazonaws.com\",\"requestParameters\":{\"userName\":\"Bob\",\"serialNumber\":\"arn:aws:iam::0123456789012:mfa/Bob\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-adea-490a-a806-EXAMPLE\",\"eventID\":\"EXAMPLE-3fdc-4b2a-9885-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", - "created": "2019-11-27T15:11:09.000Z", + "created": "2021-11-11T01:02:03.123456789Z", "kind": "event", "action": "EnableMFADevice", "id": "EXAMPLE-3fdc-4b2a-9885-EXAMPLE", diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-insight-json.log-config.yml b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-insight-json.log-config.yml deleted file mode 100644 index c39dc38617..0000000000 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-insight-json.log-config.yml +++ /dev/null @@ -1,2 +0,0 @@ -dynamic_fields: - event.ingested: ".*" diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-insight-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-insight-json.log-expected.json index 5c973ebbb8..f8a1c9b1f2 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-insight-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-insight-json.log-expected.json @@ -5,13 +5,15 @@ "region": "us-east-1" }, "@timestamp": "2020-09-09T23:00:00.000Z", + "ecs": { + "version": "1.12.0" + }, "event": { - "ingested": "2021-04-23T12:15:55.161878656Z", "original": "{\"eventVersion\":\"1.07\",\"eventTime\":\"2020-09-09T23:00:00Z\",\"awsRegion\":\"us-east-1\",\"eventID\":\"41ed77ca-d659-b45a-8e9a-74e504300007\",\"eventType\":\"AwsCloudTrailInsight\",\"recipientAccountId\":\"123456789012\",\"sharedEventID\":\"e672c2b1-e71a-4779-f96c-02da7bb30d2e\",\"insightDetails\":{\"state\":\"End\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"AttachUserPolicy\",\"insightType\":\"ApiCallRateInsight\",\"insffightContext\":{\"statistics\":{\"baseline\":{\"average\":0.0},\"insight\":{\"average\":2.0},\"insightDuration\":1,\"baselineDuration\":11459},\"attributions\":[{\"attribute\":\"userIdentityArn\",\"insight\":[{\"value\":\"arn:aws:iam::123456789012:user/Alice\",\"average\":2.0}],\"baseline\":[]},{\"attribute\":\"userAgent\",\"insight\":[{\"value\":\"console.amazonaws.com\",\"average\":2.0}],\"baseline\":[]},{\"attribute\":\"errorCode\",\"insight\":[{\"value\":\"null\",\"average\":2.0}],\"baseline\":[]}]}},\"eventCategory\":\"Insight\"}", - "created": "2020-09-09T23:00:00.000Z", - "kind": "event", "id": "41ed77ca-d659-b45a-8e9a-74e504300007", "type": "info", + "created": "2021-11-11T01:02:03.123456789Z", + "kind": "event", "outcome": "success" }, "aws": { @@ -70,7 +72,10 @@ "recipient_account_id": "123456789012", "event_category": "Insight" } - } + }, + "tags": [ + "preserve_original_event" + ] } ] } \ No newline at end of file diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-remove-user-from-group-json.log-config.yml b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-remove-user-from-group-json.log-config.yml deleted file mode 100644 index c39dc38617..0000000000 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-remove-user-from-group-json.log-config.yml +++ /dev/null @@ -1,2 +0,0 @@ -dynamic_fields: - event.ingested: ".*" diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-remove-user-from-group-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-remove-user-from-group-json.log-expected.json index fe39acb048..f3ade51a53 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-remove-user-from-group-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-remove-user-from-group-json.log-expected.json @@ -1,6 +1,13 @@ { "expected": [ { + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "tags": [ + "preserve_original_event" + ], "cloud": { "region": "us-east-1", "account": { @@ -8,21 +15,19 @@ } }, "@timestamp": "2020-01-06T15:19:50.000Z", + "ecs": { + "version": "1.12.0" + }, "related": { "user": [ "Alice", "Bob" ] }, - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, "event": { - "ingested": "2021-04-23T12:15:55.180758340Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-06T14:36:28Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-06T15:19:50Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"RemoveUserFromGroup\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"groupName\":\"Admin\",\"userName\":\"Bob\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-0bf0-47be-bc80-EXAMPLE\",\"eventID\":\"EXAMPLE-6e8b-431a-94f4-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", - "created": "2020-01-06T15:19:50.000Z", + "created": "2021-11-11T01:02:03.123456789Z", "kind": "event", "action": "RemoveUserFromGroup", "id": "EXAMPLE-6e8b-431a-94f4-EXAMPLE", diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-start-logging-json.log-config.yml b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-start-logging-json.log-config.yml deleted file mode 100644 index c39dc38617..0000000000 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-start-logging-json.log-config.yml +++ /dev/null @@ -1,2 +0,0 @@ -dynamic_fields: - event.ingested: ".*" diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-start-logging-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-start-logging-json.log-expected.json index fa90a4ca53..3101de44c8 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-start-logging-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-start-logging-json.log-expected.json @@ -1,6 +1,13 @@ { "expected": [ { + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "tags": [ + "preserve_original_event" + ], "cloud": { "region": "us-west-2", "account": { @@ -8,20 +15,18 @@ } }, "@timestamp": "2020-01-08T15:30:25.000Z", + "ecs": { + "version": "1.12.0" + }, "related": { "user": [ "Alice" ] }, - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, "event": { - "ingested": "2021-04-23T12:15:55.217310455Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"sessionIssuer\":{},\"webIdFederationData\":{},\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-08T15:12:16Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-08T15:30:25Z\",\"eventSource\":\"cloudtrail.amazonaws.com\",\"eventName\":\"StartLogging\",\"awsRegion\":\"us-west-2\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"name\":\"TEST-trail\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-1c30-4f43-9763-EXAMPLE\",\"eventID\":\"EXAMPLE-aa78-4a84-a27f-EXAMPLE\",\"readOnly\":false,\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "cloudtrail.amazonaws.com", - "created": "2020-01-08T15:30:25.000Z", + "created": "2021-11-11T01:02:03.123456789Z", "kind": "event", "action": "StartLogging", "id": "EXAMPLE-aa78-4a84-a27f-EXAMPLE", diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-stop-logging-json.log-config.yml b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-stop-logging-json.log-config.yml deleted file mode 100644 index c39dc38617..0000000000 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-stop-logging-json.log-config.yml +++ /dev/null @@ -1,2 +0,0 @@ -dynamic_fields: - event.ingested: ".*" diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-stop-logging-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-stop-logging-json.log-expected.json index ccbbcdd9f3..79d7439e30 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-stop-logging-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-stop-logging-json.log-expected.json @@ -1,6 +1,13 @@ { "expected": [ { + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "tags": [ + "preserve_original_event" + ], "cloud": { "region": "us-west-2", "account": { @@ -8,20 +15,18 @@ } }, "@timestamp": "2020-01-09T16:46:16.000Z", + "ecs": { + "version": "1.12.0" + }, "related": { "user": [ "Alice" ] }, - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, "event": { - "ingested": "2021-04-23T12:15:55.250611410Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"sessionIssuer\":{},\"webIdFederationData\":{},\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-09T16:36:17Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-09T16:46:16Z\",\"eventSource\":\"cloudtrail.amazonaws.com\",\"eventName\":\"StopLogging\",\"awsRegion\":\"us-west-2\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"name\":\"arn:aws:cloudtrail:us-west-2:0123456789012:trail/TEST-trail\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-869f-4fec-86f9-EXAMPLE\",\"eventID\":\"EXAMPLE-8cc3-42db-9a0d-EXAMPLE\",\"readOnly\":false,\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "cloudtrail.amazonaws.com", - "created": "2020-01-09T16:46:16.000Z", + "created": "2021-11-11T01:02:03.123456789Z", "kind": "event", "action": "StopLogging", "id": "EXAMPLE-8cc3-42db-9a0d-EXAMPLE", diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-access-key-json.log-config.yml b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-access-key-json.log-config.yml deleted file mode 100644 index c39dc38617..0000000000 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-access-key-json.log-config.yml +++ /dev/null @@ -1,2 +0,0 @@ -dynamic_fields: - event.ingested: ".*" diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-access-key-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-access-key-json.log-expected.json index efb8442615..0d57cca9cf 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-access-key-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-access-key-json.log-expected.json @@ -1,6 +1,13 @@ { "expected": [ { + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "tags": [ + "preserve_original_event" + ], "cloud": { "region": "us-east-1", "account": { @@ -8,21 +15,19 @@ } }, "@timestamp": "2020-01-10T15:01:23.000Z", + "ecs": { + "version": "1.12.0" + }, "related": { "user": [ "Alice", "Bob" ] }, - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, "event": { - "ingested": "2021-04-23T12:15:55.283645028Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-10T14:38:30Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-10T15:01:23Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"UpdateAccessKey\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"status\":\"Inactive\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Bob\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-7d0c-45f4-b25b-EXAMPLE\",\"eventID\":\"EXAMPLE-0ef0-42cd-8551-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", - "created": "2020-01-10T15:01:23.000Z", + "created": "2021-11-11T01:02:03.123456789Z", "kind": "event", "action": "UpdateAccessKey", "id": "EXAMPLE-0ef0-42cd-8551-EXAMPLE", diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-accout-password-policy-json.log-config.yml b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-accout-password-policy-json.log-config.yml deleted file mode 100644 index c39dc38617..0000000000 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-accout-password-policy-json.log-config.yml +++ /dev/null @@ -1,2 +0,0 @@ -dynamic_fields: - event.ingested: ".*" diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-accout-password-policy-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-accout-password-policy-json.log-expected.json index 22c064cf00..cdc24c9d2f 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-accout-password-policy-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-accout-password-policy-json.log-expected.json @@ -1,6 +1,13 @@ { "expected": [ { + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "tags": [ + "preserve_original_event" + ], "cloud": { "region": "us-east-1", "account": { @@ -8,20 +15,18 @@ } }, "@timestamp": "2020-01-10T18:05:33.000Z", + "ecs": { + "version": "1.12.0" + }, "related": { "user": [ "Alice" ] }, - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, "event": { - "ingested": "2021-04-23T12:15:55.317715645Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-10T14:38:30Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-10T18:05:33Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"UpdateAccountPasswordPolicy\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"requireLowercaseCharacters\":true,\"requireSymbols\":true,\"requireNumbers\":true,\"minimumPasswordLength\":12,\"requireUppercaseCharacters\":true,\"allowUsersToChangePassword\":true},\"responseElements\":null,\"requestID\":\"EXAMPLE-5ebf-4bc3-a349-EXAMPLE\",\"eventID\":\"EXAMPLE-91f9-49f3-948c-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", - "created": "2020-01-10T18:05:33.000Z", + "created": "2021-11-11T01:02:03.123456789Z", "kind": "event", "action": "UpdateAccountPasswordPolicy", "id": "EXAMPLE-91f9-49f3-948c-EXAMPLE", diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-group-json.log-config.yml b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-group-json.log-config.yml deleted file mode 100644 index c39dc38617..0000000000 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-group-json.log-config.yml +++ /dev/null @@ -1,2 +0,0 @@ -dynamic_fields: - event.ingested: ".*" diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-group-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-group-json.log-expected.json index 297780c6a2..7b75f3b3ec 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-group-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-group-json.log-expected.json @@ -1,6 +1,13 @@ { "expected": [ { + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "tags": [ + "preserve_original_event" + ], "cloud": { "region": "us-east-1", "account": { @@ -8,20 +15,18 @@ } }, "@timestamp": "2020-01-09T02:23:11.000Z", + "ecs": { + "version": "1.12.0" + }, "related": { "user": [ "Alice" ] }, - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, "event": { - "ingested": "2021-04-23T12:15:55.350927729Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"0123456789012\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\"},\"eventTime\":\"2020-01-09T02:23:11Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"UpdateGroup\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46\",\"requestParameters\":{\"newGroupName\":\"TEST-GROUP2\",\"groupName\":\"TEST-GROUP\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-c22d-4fca-b40a-EXAMPLE\",\"eventID\":\"EXAMPLE-c3aa-487b-b05e-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", - "created": "2020-01-09T02:23:11.000Z", + "created": "2021-11-11T01:02:03.123456789Z", "kind": "event", "action": "UpdateGroup", "id": "EXAMPLE-c3aa-487b-b05e-EXAMPLE", @@ -70,6 +75,13 @@ } }, { + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "tags": [ + "preserve_original_event" + ], "cloud": { "region": "us-east-1", "account": { @@ -77,20 +89,18 @@ } }, "@timestamp": "2020-01-09T02:24:35.000Z", + "ecs": { + "version": "1.12.0" + }, "related": { "user": [ "Alice" ] }, - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, "event": { - "ingested": "2021-04-23T12:15:55.350935229Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"0123456789012\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\"},\"eventTime\":\"2020-01-09T02:24:35Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"UpdateGroup\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46\",\"errorCode\":\"EntityAlreadyExistsException\",\"errorMessage\":\"Group with name TEST-GROUP already exists.\",\"requestParameters\":{\"newGroupName\":\"TEST-GROUP\",\"groupName\":\"TEST-GROUP2\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-f673-4ce7-8529-EXAMPLE\",\"eventID\":\"EXAMPLE-6a0b-475c-b5db-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", - "created": "2020-01-09T02:24:35.000Z", + "created": "2021-11-11T01:02:03.123456789Z", "kind": "event", "action": "UpdateGroup", "id": "EXAMPLE-6a0b-475c-b5db-EXAMPLE", diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-login-profile-json.log-config.yml b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-login-profile-json.log-config.yml deleted file mode 100644 index c39dc38617..0000000000 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-login-profile-json.log-config.yml +++ /dev/null @@ -1,2 +0,0 @@ -dynamic_fields: - event.ingested: ".*" diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-login-profile-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-login-profile-json.log-expected.json index 395dd48f5e..54e213526e 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-login-profile-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-login-profile-json.log-expected.json @@ -1,6 +1,13 @@ { "expected": [ { + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "tags": [ + "preserve_original_event" + ], "cloud": { "region": "us-east-1", "account": { @@ -8,21 +15,19 @@ } }, "@timestamp": "2020-01-10T18:25:42.000Z", + "ecs": { + "version": "1.12.0" + }, "related": { "user": [ "Alice", "Bob" ] }, - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, "event": { - "ingested": "2021-04-23T12:15:55.413508601Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-10T14:38:30Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-10T18:25:42Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"UpdateLoginProfile\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"userName\":\"Bob\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-0dc6-447a-8859-EXAMPLE\",\"eventID\":\"EXAMPLE-c3b6-4498-b818-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", - "created": "2020-01-10T18:25:42.000Z", + "created": "2021-11-11T01:02:03.123456789Z", "kind": "event", "action": "UpdateLoginProfile", "id": "EXAMPLE-c3b6-4498-b818-EXAMPLE", diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-ssh-public-key-json.log-config.yml b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-ssh-public-key-json.log-config.yml deleted file mode 100644 index c39dc38617..0000000000 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-ssh-public-key-json.log-config.yml +++ /dev/null @@ -1,2 +0,0 @@ -dynamic_fields: - event.ingested: ".*" diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-ssh-public-key-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-ssh-public-key-json.log-expected.json index e8e85e3169..c6ee9087cd 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-ssh-public-key-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-ssh-public-key-json.log-expected.json @@ -1,6 +1,13 @@ { "expected": [ { + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "tags": [ + "preserve_original_event" + ], "cloud": { "region": "us-east-1", "account": { @@ -8,21 +15,19 @@ } }, "@timestamp": "2020-01-10T16:06:54.000Z", + "ecs": { + "version": "1.12.0" + }, "related": { "user": [ "Alice", "Bob" ] }, - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, "event": { - "ingested": "2021-04-23T12:15:55.459387853Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-10T14:38:30Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-10T16:06:54Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"UpdateSSHPublicKey\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"status\":\"Inactive\",\"userName\":\"Bob\",\"sSHPublicKeyId\":\"EXAMPLE_KEY_ID\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-32f3-4a92-82e1-EXAMPLE\",\"eventID\":\"EXAMPLE-5c88-4652-9ee9-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", - "created": "2020-01-10T16:06:54.000Z", + "created": "2021-11-11T01:02:03.123456789Z", "kind": "event", "action": "UpdateSSHPublicKey", "id": "EXAMPLE-5c88-4652-9ee9-EXAMPLE", @@ -76,6 +81,13 @@ } }, { + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "tags": [ + "preserve_original_event" + ], "cloud": { "region": "us-east-1", "account": { @@ -83,21 +95,19 @@ } }, "@timestamp": "2020-01-10T16:06:54.000Z", + "ecs": { + "version": "1.12.0" + }, "related": { "user": [ "Alice", "Bob" ] }, - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, "event": { - "ingested": "2021-04-23T12:15:55.459396392Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-10T14:38:30Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-10T16:06:54Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"UpdateSSHPublicKey\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"status\":\"Inactive\",\"userName\":\"Bob\",\"sSHPublicKeyId\":\"EXAMPLE_KEY_ID\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-32f3-4a92-82e1-EXAMPLE\",\"eventID\":\"EXAMPLE-5c88-4652-9ee9-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", - "created": "2020-01-10T16:06:54.000Z", + "created": "2021-11-11T01:02:03.123456789Z", "kind": "event", "action": "UpdateSSHPublicKey", "id": "EXAMPLE-5c88-4652-9ee9-EXAMPLE", diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-trail-json.log b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-trail-json.log index f8a9bc9e2a..9b440298c6 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-trail-json.log +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-trail-json.log @@ -1,2 +1,2 @@ -{"eventVersion":"1.04","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice"},"eventTime":"2016-07-14T19:15:45Z","eventSource":"cloudtrail.amazonaws.com","eventName":"UpdateTrail","awsRegion":"us-east-2","sourceIPAddress":"205.251.233.182","userAgent":"aws-cli/1.10.32 Python/2.7.9 Windows/7 botocore/1.4.22","errorCode":"TrailNotFoundException","errorMessage":"Unknown trail: myTrail2 for the user: 123456789012","requestParameters":{"name":"myTrail2"},"responseElements":null,"requestID":"5d40662a-49f7-11e6-97e4-dEXAMPLE","eventID":"b7d4398e-b2f0-4faa-9c76-e2EXAMPLE","eventType":"AwsApiCall","recipientAccountId":"123456789012"} +{"eventVersion":"1.04","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice"},"eventTime":"2016-07-14T19:15:45Z","eventSource":"cloudtrail.amazonaws.com","eventName":"UpdateTrail","awsRegion":"us-east-2","sourceIPAddress":"89.160.20.156","userAgent":"aws-cli/1.10.32 Python/2.7.9 Windows/7 botocore/1.4.22","errorCode":"TrailNotFoundException","errorMessage":"Unknown trail: myTrail2 for the user: 123456789012","requestParameters":{"name":"myTrail2"},"responseElements":null,"requestID":"5d40662a-49f7-11e6-97e4-dEXAMPLE","eventID":"b7d4398e-b2f0-4faa-9c76-e2EXAMPLE","eventType":"AwsApiCall","recipientAccountId":"123456789012"} {"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"EXAMPLE_ID","arn":"arn:aws:iam::0123456789012:user/Alice","accountId":"0123456789012","accessKeyId":"EXAMPLE_KEY","userName":"Alice","sessionContext":{"sessionIssuer":{},"webIdFederationData":{},"attributes":{"mfaAuthenticated":"true","creationDate":"2020-01-08T15:12:16Z"}},"invokedBy":"signin.amazonaws.com"},"eventTime":"2020-01-08T20:58:45Z","eventSource":"cloudtrail.amazonaws.com","eventName":"UpdateTrail","awsRegion":"us-west-2","sourceIPAddress":"127.0.0.1","userAgent":"signin.amazonaws.com","requestParameters":{"name":"arn:aws:cloudtrail:us-west-2:0123456789012:trail/TEST-trail","s3BucketName":"test-cloudtrail-bucket","snsTopicName":"","isMultiRegionTrail":true,"enableLogFileValidation":false,"kmsKeyId":""},"responseElements":{"name":"TEST-trail","s3BucketName":"test-cloudtrail-bucket","snsTopicName":"","snsTopicARN":"","includeGlobalServiceEvents":true,"isMultiRegionTrail":true,"trailARN":"arn:aws:cloudtrail:us-west-2:0123456789012:trail/TEST-trail","logFileValidationEnabled":false,"isOrganizationTrail":false},"requestID":"EXAMPLE-f3da-42d1-84f5-EXAMPLE","eventID":"EXAMPLE-b5e9-4846-8407-EXAMPLE","readOnly":false,"eventType":"AwsApiCall","recipientAccountId":"0123456789012"} diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-trail-json.log-config.yml b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-trail-json.log-config.yml deleted file mode 100644 index c39dc38617..0000000000 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-trail-json.log-config.yml +++ /dev/null @@ -1,2 +0,0 @@ -dynamic_fields: - event.ingested: ".*" diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-trail-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-trail-json.log-expected.json index b9133a03e6..754605022e 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-trail-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-trail-json.log-expected.json @@ -1,6 +1,31 @@ { "expected": [ { + "source": { + "geo": { + "continent_name": "Europe", + "region_iso_code": "SE-AB", + "city_name": "Tumba", + "country_iso_code": "SE", + "country_name": "Sweden", + "region_name": "Stockholm", + "location": { + "lon": 17.8167, + "lat": 59.2 + } + }, + "as": { + "number": 29518, + "organization": { + "name": "Bredband2 AB" + } + }, + "address": "89.160.20.156", + "ip": "89.160.20.156" + }, + "tags": [ + "preserve_original_event" + ], "cloud": { "region": "us-east-2", "account": { @@ -8,38 +33,18 @@ } }, "@timestamp": "2016-07-14T19:15:45.000Z", + "ecs": { + "version": "1.12.0" + }, "related": { "user": [ "Alice" ] }, - "source": { - "geo": { - "continent_name": "North America", - "region_iso_code": "US-OR", - "city_name": "Boardman", - "country_iso_code": "US", - "country_name": "United States", - "region_name": "Oregon", - "location": { - "lon": -119.7143, - "lat": 45.8491 - } - }, - "as": { - "number": 16509, - "organization": { - "name": "Amazon.com, Inc." - } - }, - "address": "205.251.233.182", - "ip": "205.251.233.182" - }, "event": { - "ingested": "2021-04-23T12:15:55.523468018Z", - "original": "{\"eventVersion\":\"1.04\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EX_PRINCIPAL_ID\",\"arn\":\"arn:aws:iam::123456789012:user/Alice\",\"accountId\":\"123456789012\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Alice\"},\"eventTime\":\"2016-07-14T19:15:45Z\",\"eventSource\":\"cloudtrail.amazonaws.com\",\"eventName\":\"UpdateTrail\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"205.251.233.182\",\"userAgent\":\"aws-cli/1.10.32 Python/2.7.9 Windows/7 botocore/1.4.22\",\"errorCode\":\"TrailNotFoundException\",\"errorMessage\":\"Unknown trail: myTrail2 for the user: 123456789012\",\"requestParameters\":{\"name\":\"myTrail2\"},\"responseElements\":null,\"requestID\":\"5d40662a-49f7-11e6-97e4-dEXAMPLE\",\"eventID\":\"b7d4398e-b2f0-4faa-9c76-e2EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"123456789012\"}", + "original": "{\"eventVersion\":\"1.04\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EX_PRINCIPAL_ID\",\"arn\":\"arn:aws:iam::123456789012:user/Alice\",\"accountId\":\"123456789012\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Alice\"},\"eventTime\":\"2016-07-14T19:15:45Z\",\"eventSource\":\"cloudtrail.amazonaws.com\",\"eventName\":\"UpdateTrail\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"89.160.20.156\",\"userAgent\":\"aws-cli/1.10.32 Python/2.7.9 Windows/7 botocore/1.4.22\",\"errorCode\":\"TrailNotFoundException\",\"errorMessage\":\"Unknown trail: myTrail2 for the user: 123456789012\",\"requestParameters\":{\"name\":\"myTrail2\"},\"responseElements\":null,\"requestID\":\"5d40662a-49f7-11e6-97e4-dEXAMPLE\",\"eventID\":\"b7d4398e-b2f0-4faa-9c76-e2EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"123456789012\"}", "provider": "cloudtrail.amazonaws.com", - "created": "2016-07-14T19:15:45.000Z", + "created": "2021-11-11T01:02:03.123456789Z", "kind": "event", "action": "UpdateTrail", "id": "b7d4398e-b2f0-4faa-9c76-e2EXAMPLE", @@ -83,6 +88,13 @@ } }, { + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "tags": [ + "preserve_original_event" + ], "cloud": { "region": "us-west-2", "account": { @@ -90,20 +102,18 @@ } }, "@timestamp": "2020-01-08T20:58:45.000Z", + "ecs": { + "version": "1.12.0" + }, "related": { "user": [ "Alice" ] }, - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, "event": { - "ingested": "2021-04-23T12:15:55.523477094Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"sessionIssuer\":{},\"webIdFederationData\":{},\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-08T15:12:16Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-08T20:58:45Z\",\"eventSource\":\"cloudtrail.amazonaws.com\",\"eventName\":\"UpdateTrail\",\"awsRegion\":\"us-west-2\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"name\":\"arn:aws:cloudtrail:us-west-2:0123456789012:trail/TEST-trail\",\"s3BucketName\":\"test-cloudtrail-bucket\",\"snsTopicName\":\"\",\"isMultiRegionTrail\":true,\"enableLogFileValidation\":false,\"kmsKeyId\":\"\"},\"responseElements\":{\"name\":\"TEST-trail\",\"s3BucketName\":\"test-cloudtrail-bucket\",\"snsTopicName\":\"\",\"snsTopicARN\":\"\",\"includeGlobalServiceEvents\":true,\"isMultiRegionTrail\":true,\"trailARN\":\"arn:aws:cloudtrail:us-west-2:0123456789012:trail/TEST-trail\",\"logFileValidationEnabled\":false,\"isOrganizationTrail\":false},\"requestID\":\"EXAMPLE-f3da-42d1-84f5-EXAMPLE\",\"eventID\":\"EXAMPLE-b5e9-4846-8407-EXAMPLE\",\"readOnly\":false,\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "cloudtrail.amazonaws.com", - "created": "2020-01-08T20:58:45.000Z", + "created": "2021-11-11T01:02:03.123456789Z", "kind": "event", "action": "UpdateTrail", "id": "EXAMPLE-b5e9-4846-8407-EXAMPLE", diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-user-json.log b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-user-json.log index 62721399a4..f4ec7b890a 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-user-json.log +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-user-json.log @@ -1,2 +1 @@ -{"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice"},"eventTime":"2020-01-08T20:53:12Z","eventSource":"iam.amazonaws.com","eventName":"UpdateUser","awsRegion":"us-east-1","sourceIPAddress":"127.0.0.1","userAgent":"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46","requestParameters":{"userName":"Bob","newUserName":"Robert"},"responseElements":null,"requestID":"3a6b3260-739d-465e-9406-bcEXAMPLE","eventID":"9150d546-3564-4262-8e62-110EXAMPLE","eventType":"AwsApiCall","recipientAccountId":"123456789012"} - +{"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice"},"eventTime":"2020-01-08T20:53:12Z","eventSource":"iam.amazonaws.com","eventName":"UpdateUser","awsRegion":"us-east-1","sourceIPAddress":"127.0.0.1","userAgent":"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46","requestParameters":{"userName":"Bob","newUserName":"Robert"},"responseElements":null,"requestID":"3a6b3260-739d-465e-9406-bcEXAMPLE","eventID":"9150d546-3564-4262-8e62-110EXAMPLE","eventType":"AwsApiCall","recipientAccountId":"123456789012"} \ No newline at end of file diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-user-json.log-config.yml b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-user-json.log-config.yml deleted file mode 100644 index c39dc38617..0000000000 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-user-json.log-config.yml +++ /dev/null @@ -1,2 +0,0 @@ -dynamic_fields: - event.ingested: ".*" diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-user-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-user-json.log-expected.json index 7f1b90e98e..e324600a27 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-user-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-user-json.log-expected.json @@ -1,6 +1,13 @@ { "expected": [ { + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "tags": [ + "preserve_original_event" + ], "cloud": { "region": "us-east-1", "account": { @@ -8,6 +15,9 @@ } }, "@timestamp": "2020-01-08T20:53:12.000Z", + "ecs": { + "version": "1.12.0" + }, "related": { "user": [ "Alice", @@ -15,15 +25,10 @@ "Robert" ] }, - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, "event": { - "ingested": "2021-04-23T12:15:55.609811323Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EX_PRINCIPAL_ID\",\"arn\":\"arn:aws:iam::123456789012:user/Alice\",\"accountId\":\"123456789012\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Alice\"},\"eventTime\":\"2020-01-08T20:53:12Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"UpdateUser\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46\",\"requestParameters\":{\"userName\":\"Bob\",\"newUserName\":\"Robert\"},\"responseElements\":null,\"requestID\":\"3a6b3260-739d-465e-9406-bcEXAMPLE\",\"eventID\":\"9150d546-3564-4262-8e62-110EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"123456789012\"}", "provider": "iam.amazonaws.com", - "created": "2020-01-08T20:53:12.000Z", + "created": "2021-11-11T01:02:03.123456789Z", "kind": "event", "action": "UpdateUser", "id": "9150d546-3564-4262-8e62-110EXAMPLE", @@ -73,13 +78,6 @@ }, "version": "1.16.310" } - }, - { - "event": { - "type": "info", - "ingested": "2021-04-23T12:15:55.609819468Z", - "kind": "event" - } } ] } \ No newline at end of file diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-upload-ssh-public-key-json.log-config.yml b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-upload-ssh-public-key-json.log-config.yml deleted file mode 100644 index c39dc38617..0000000000 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-upload-ssh-public-key-json.log-config.yml +++ /dev/null @@ -1,2 +0,0 @@ -dynamic_fields: - event.ingested: ".*" diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-upload-ssh-public-key-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-upload-ssh-public-key-json.log-expected.json index 399c7f10b3..63e08a3cb3 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-upload-ssh-public-key-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-upload-ssh-public-key-json.log-expected.json @@ -1,6 +1,13 @@ { "expected": [ { + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "tags": [ + "preserve_original_event" + ], "cloud": { "region": "us-east-1", "account": { @@ -8,20 +15,18 @@ } }, "@timestamp": "2020-01-10T16:06:40.000Z", + "ecs": { + "version": "1.12.0" + }, "related": { "user": [ "Alice" ] }, - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, "event": { - "ingested": "2021-04-23T12:15:55.649407821Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-10T14:38:30Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-10T16:06:40Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"UploadSSHPublicKey\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"sSHPublicKeyBody\":\"ssh-rsa AAAAdeadcodedeadcode Alice@localhost.domain\",\"userName\":\"Alice\"},\"responseElements\":{\"sSHPublicKey\":{\"fingerprint\":\"de:ad:c0:de:de:ad:c0:de:de:ad:c0:de:de:ad:c0:de\",\"status\":\"Active\",\"uploadDate\":\"Jan 10, 2020 4:06:40 PM\",\"userName\":\"Alice\",\"sSHPublicKeyId\":\"EXAMPLE_KEY_ID\",\"sSHPublicKeyBody\":\"ssh-rsa AAAAdeadcodedeadcode Alice@localhost.domain\"}},\"requestID\":\"EXAMPLE-44b9-41cd-90f2-EXAMPLE\",\"eventID\":\"EXAMPLE-9a9d-4da4-9998-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", - "created": "2020-01-10T16:06:40.000Z", + "created": "2021-11-11T01:02:03.123456789Z", "kind": "event", "action": "UploadSSHPublicKey", "id": "EXAMPLE-9a9d-4da4-9998-EXAMPLE", diff --git a/test/packages/aws/data_stream/cloudtrail/agent/stream/aws-s3.yml.hbs b/test/packages/aws/data_stream/cloudtrail/agent/stream/aws-s3.yml.hbs new file mode 100644 index 0000000000..50ef9090b8 --- /dev/null +++ b/test/packages/aws/data_stream/cloudtrail/agent/stream/aws-s3.yml.hbs @@ -0,0 +1,64 @@ +queue_url: {{queue_url}} +file_selectors: +{{#if cloudtrail_regex}} + - regex: {{cloudtrail_regex}} + expand_event_list_from_field: 'Records' +{{/if}} +{{#if cloudtrail_digest_regex}} + - regex: {{cloudtrail_digest_regex}} +{{/if}} +{{#if cloudtrail_insight_regex}} + - regex: {{cloudtrail_insight_regex}} + expand_event_list_from_field: 'Records' +{{/if}} +expand_event_list_from_field: Records +{{#if credential_profile_name}} +credential_profile_name: {{credential_profile_name}} +{{/if}} +{{#if shared_credential_file}} +shared_credential_file: {{shared_credential_file}} +{{/if}} +{{#if visibility_timeout}} +visibility_timeout: {{visibility_timeout}} +{{/if}} +{{#if api_timeout}} +api_timeout: {{api_timeout}} +{{/if}} +{{#if max_number_of_messages}} +max_number_of_messages: {{max_number_of_messages}} +{{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} +{{#if access_key_id}} +access_key_id: {{access_key_id}} +{{/if}} +{{#if secret_access_key}} +secret_access_key: {{secret_access_key}} +{{/if}} +{{#if session_token}} +session_token: {{session_token}} +{{/if}} +{{#if role_arn}} +role_arn: {{role_arn}} +{{/if}} +{{#if fips_enabled}} +fips_enabled: {{fips_enabled}} +{{/if}} +{{#if proxy_url }} +proxy_url: {{proxy_url}} +{{/if}} +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} \ No newline at end of file diff --git a/test/packages/aws/data_stream/cloudtrail/agent/stream/httpjson.yml.hbs b/test/packages/aws/data_stream/cloudtrail/agent/stream/httpjson.yml.hbs new file mode 100644 index 0000000000..27d1775b51 --- /dev/null +++ b/test/packages/aws/data_stream/cloudtrail/agent/stream/httpjson.yml.hbs @@ -0,0 +1,63 @@ +config_version: 2 +interval: {{interval}} +{{#unless token}} +{{#if username}} +{{#if password}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +{{/if}} +{{/if}} +{{/unless}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +{{#unless username}} +{{#unless password}} +{{#if token}} + - set: + target: header.Authorization + value: {{token}} +{{/if}} +{{/unless}} +{{/unless}} +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} \ No newline at end of file diff --git a/test/packages/aws/data_stream/cloudtrail/elasticsearch/ingest_pipeline/default.yml b/test/packages/aws/data_stream/cloudtrail/elasticsearch/ingest_pipeline/default.yml index 3e0000d26f..bbc66f7fa8 100644 --- a/test/packages/aws/data_stream/cloudtrail/elasticsearch/ingest_pipeline/default.yml +++ b/test/packages/aws/data_stream/cloudtrail/elasticsearch/ingest_pipeline/default.yml @@ -1,24 +1,28 @@ --- description: Pipeline for AWS CloudTrail Logs processors: - - set: - field: event.ingested - value: '{{_ingest.timestamp}}' - rename: - field: "message" - target_field: "event.original" + field: message + target_field: event.original + - set: + if: ctx['@timestamp'] != null + field: event.created + copy_from: '@timestamp' - json: - field: "event.original" - target_field: "json" + field: event.original + target_field: json + - pipeline: + if: ctx?.json?.preview != null + name: '{{ IngestPipeline "third-party" }}' + - set: + field: ecs.version + value: '1.12.0' - date: - field: "json.eventTime" + field: json.eventTime target_field: "@timestamp" ignore_failure: true formats: - ISO8601 - - set: - field: event.created - value: '{{@timestamp}}' - script: description: Drops null/empty values recursively lang: painless @@ -37,93 +41,93 @@ processors: } drop(ctx); - rename: - field: "json.eventVersion" - target_field: "aws.cloudtrail.event_version" + field: json.eventVersion + target_field: aws.cloudtrail.event_version ignore_failure: true - rename: - field: "json.userIdentity.type" - target_field: "aws.cloudtrail.user_identity.type" + field: json.userIdentity.type + target_field: aws.cloudtrail.user_identity.type ignore_failure: true - append: field: related.user value: '{{json.userIdentity.userName}}' allow_duplicates: false - if: 'ctx.json?.userIdentity?.userName != null' + if: ctx.json?.userIdentity?.userName != null - rename: - field: "json.userIdentity.userName" - target_field: "user.name" + field: json.userIdentity.userName + target_field: user.name ignore_failure: true - rename: - field: "json.userIdentity.principalId" - target_field: "user.id" + field: json.userIdentity.principalId + target_field: user.id ignore_failure: true - rename: - field: "json.userIdentity.arn" - target_field: "aws.cloudtrail.user_identity.arn" + field: json.userIdentity.arn + target_field: aws.cloudtrail.user_identity.arn ignore_failure: true - rename: - field: "json.userIdentity.accountId" - target_field: "cloud.account.id" + field: json.userIdentity.accountId + target_field: cloud.account.id ignore_failure: true - rename: - field: "json.userIdentity.accessKeyId" - target_field: "aws.cloudtrail.user_identity.access_key_id" + field: json.userIdentity.accessKeyId + target_field: aws.cloudtrail.user_identity.access_key_id ignore_failure: true - rename: - field: "json.userIdentity.sessionContext.attributes.mfaAuthenticated" - target_field: "aws.cloudtrail.user_identity.session_context.mfa_authenticated" + field: json.userIdentity.sessionContext.attributes.mfaAuthenticated + target_field: aws.cloudtrail.user_identity.session_context.mfa_authenticated ignore_failure: true - date: - field: "json.userIdentity.sessionContext.attributes.creationDate" - target_field: "aws.cloudtrail.user_identity.session_context.creation_date" + field: json.userIdentity.sessionContext.attributes.creationDate + target_field: aws.cloudtrail.user_identity.session_context.creation_date ignore_failure: true formats: - ISO8601 - rename: - field: "json.userIdentity.sessionContext.sessionIssuer.type" - target_field: "aws.cloudtrail.user_identity.session_context.session_issuer.type" + field: json.userIdentity.sessionContext.sessionIssuer.type + target_field: aws.cloudtrail.user_identity.session_context.session_issuer.type ignore_failure: true # userIdentity.sessionIssuer.userName is only set with assumed roles. - rename: - field: "json.userIdentity.sessionContext.sessionIssuer.userName" - target_field: "user.name" + field: json.userIdentity.sessionContext.sessionIssuer.userName + target_field: user.name ignore_failure: true - rename: - field: "json.userIdentity.sessionContext.sessionIssuer.principalId" - target_field: "aws.cloudtrail.user_identity.session_context.session_issuer.principal_id" + field: json.userIdentity.sessionContext.sessionIssuer.principalId + target_field: aws.cloudtrail.user_identity.session_context.session_issuer.principal_id ignore_failure: true - rename: - field: "json.userIdentity.sessionContext.sessionIssuer.arn" - target_field: "aws.cloudtrail.user_identity.session_context.session_issuer.arn" + field: json.userIdentity.sessionContext.sessionIssuer.arn + target_field: aws.cloudtrail.user_identity.session_context.session_issuer.arn ignore_failure: true - rename: - field: "json.userIdentity.sessionContext.sessionIssuer.accountId" - target_field: "aws.cloudtrail.user_identity.session_context.session_issuer.account_id" + field: json.userIdentity.sessionContext.sessionIssuer.accountId + target_field: aws.cloudtrail.user_identity.session_context.session_issuer.account_id ignore_failure: true - rename: - field: "json.userIdentity.invokedBy" - target_field: "aws.cloudtrail.user_identity.invoked_by" + field: json.userIdentity.invokedBy + target_field: aws.cloudtrail.user_identity.invoked_by ignore_failure: true - rename: - field: "json.eventSource" - target_field: "event.provider" + field: json.eventSource + target_field: event.provider ignore_failure: true - set: - field: "event.action" - value: "{{json.eventName}}" + field: event.action + value: '{{json.eventName}}' ignore_failure: true ignore_empty_value: true - rename: - field: "json.eventCategory" - target_field: "aws.cloudtrail.event_category" + field: json.eventCategory + target_field: aws.cloudtrail.event_category ignore_failure: true - rename: - field: "json.awsRegion" - target_field: "cloud.region" + field: json.awsRegion + target_field: cloud.region ignore_failure: true - rename: - field: "json.sourceIPAddress" - target_field: "source.address" + field: json.sourceIPAddress + target_field: source.address ignore_failure: true - grok: field: source.address @@ -131,8 +135,8 @@ processors: patterns: - ^%{IP:source.ip}$ - geoip: - field: "source.ip" - target_field: "source.geo" + field: source.ip + target_field: source.geo ignore_failure: true ignore_missing: true - geoip: @@ -152,20 +156,20 @@ processors: target_field: source.as.organization.name ignore_missing: true - user_agent: - field: "json.userAgent" - target_field: "user_agent" + field: json.userAgent + target_field: user_agent on_failure: - rename: - field: "json.userAgent" - target_field: "user_agent.original" + field: json.userAgent + target_field: user_agent.original ignore_failure: true - rename: - field: "json.errorCode" - target_field: "aws.cloudtrail.error_code" + field: json.errorCode + target_field: aws.cloudtrail.error_code ignore_failure: true - rename: - field: "json.errorMessage" - target_field: "aws.cloudtrail.error_message" + field: json.errorMessage + target_field: aws.cloudtrail.error_message ignore_failure: true - script: lang: painless @@ -200,63 +204,63 @@ processors: } ignore_failure: true - rename: - field: "json.requestId" - target_field: "aws.cloudtrail.request_id" + field: json.requestId + target_field: aws.cloudtrail.request_id ignore_failure: true - rename: - field: "json.eventID" + field: json.eventID target_field: event.id ignore_failure: true - rename: - field: "json.eventType" - target_field: "aws.cloudtrail.event_type" + field: json.eventType + target_field: aws.cloudtrail.event_type ignore_failure: true - rename: - field: "json.apiVersion" - target_field: "aws.cloudtrail.api_version" + field: json.apiVersion + target_field: aws.cloudtrail.api_version ignore_failure: true - rename: - field: "json.managementEvent" - target_field: "aws.cloudtrail.management_event" + field: json.managementEvent + target_field: aws.cloudtrail.management_event ignore_failure: true - rename: - field: "json.readOnly" - target_field: "aws.cloudtrail.read_only" + field: json.readOnly + target_field: aws.cloudtrail.read_only ignore_failure: true - rename: - field: "json.resources.ARN" - target_field: "aws.cloudtrail.resources.arn" + field: json.resources.ARN + target_field: aws.cloudtrail.resources.arn ignore_failure: true - rename: - field: "json.resources.accountId" - target_field: "aws.cloudtrail.resources.account_id" + field: json.resources.accountId + target_field: aws.cloudtrail.resources.account_id ignore_failure: true - rename: - field: "json.resources.type" - target_field: "aws.cloudtrail.resources.type" + field: json.resources.type + target_field: aws.cloudtrail.resources.type ignore_failure: true - rename: - field: "json.recipientAccountId" - target_field: "aws.cloudtrail.recipient_account_id" + field: json.recipientAccountId + target_field: aws.cloudtrail.recipient_account_id ignore_failure: true - rename: - field: "json.sharedEventId" - target_field: "aws.cloudtrail.shared_event_id" + field: json.sharedEventId + target_field: aws.cloudtrail.shared_event_id ignore_failure: true - rename: - field: "json.vpcEndpointId" - target_field: "aws.cloudtrail.vpc_endpoint_id" + field: json.vpcEndpointId + target_field: aws.cloudtrail.vpc_endpoint_id ignore_failure: true - append: field: related.user value: '{{aws.cloudtrail.flattened.request_parameters.userName}}' allow_duplicates: false - if: 'ctx.aws?.cloudtrail?.flattened?.request_parameters?.userName != null' + if: ctx.aws?.cloudtrail?.flattened?.request_parameters?.userName != null - append: field: related.user value: '{{aws.cloudtrail.flattened.request_parameters.newUserName}}' allow_duplicates: false - if: 'ctx.aws?.cloudtrail?.flattened?.request_parameters?.newUserName != null' + if: ctx.aws?.cloudtrail?.flattened?.request_parameters?.newUserName != null - script: lang: painless ignore_failure: true @@ -289,7 +293,7 @@ processors: } - script: lang: painless - ignore_failure: true + tag: Add ECS categorization params: AddUserToGroup: category: @@ -603,7 +607,7 @@ processors: ctx.event.kind = 'event'; ctx.event.type = 'info'; - if (ctx.aws.cloudtrail.error_code != null || ctx.aws.cloudtrail.error_message != null) { + if (ctx?.aws?.cloudtrail?.error_code != null || ctx?.aws?.cloudtrail?.error_message != null) { ctx.event.outcome = 'failure' } else { ctx.event.outcome = 'success' @@ -617,83 +621,87 @@ processors: ctx.event.outcome = Processors.lowercase(ctx.aws.cloudtrail.flattened.response_elements.ConsoleLogin); } + if (params.get(ctx.event.action) == null) { + return; + } + def hm = new HashMap(params.get(ctx.event.action)); hm.forEach((k, v) -> ctx.event[k] = v); - rename: - field: "json.awsAccountId" - target_field: "cloud.account.id" + field: json.awsAccountId + target_field: cloud.account.id ignore_failure: true - rename: - field: "json.previousDigestS3Object" - target_field: "file.path" + field: json.previousDigestS3Object + target_field: file.path ignore_failure: true - rename: - field: "json.previousDigestSignature" - target_field: "file.hash.sha256" + field: json.previousDigestSignature + target_field: file.hash.sha256 if: >- - ctx?.json?.previousDigestHashAlgorithm != null && ctx.json.previousDigestHashAlgorithm == 'SHA-256' + ctx.json?.previousDigestHashAlgorithm != null && ctx.json?.previousDigestHashAlgorithm == 'SHA-256' - append: - field: "related.hash" - value: "{{file.hash.sha256}}" - if: "ctx?.file?.hash?.sha256 != null" + field: related.hash + value: '{{file.hash.sha256}}' + if: ctx.file?.hash?.sha256 != null - rename: - field: "json.logFiles" - target_field: "aws.cloudtrail.digest.log_files" + field: json.logFiles + target_field: aws.cloudtrail.digest.log_files ignore_failure: true - date: - field: "json.digestStartTime" - target_field: "aws.cloudtrail.digest.start_time" + field: json.digestStartTime + target_field: aws.cloudtrail.digest.start_time ignore_failure: true formats: - ISO8601 - date: - field: "json.digestEndTime" + field: json.digestEndTime target_field: "@timestamp" ignore_failure: true formats: - ISO8601 - date: - field: "json.digestEndTime" - target_field: "aws.cloudtrail.digest.end_time" + field: json.digestEndTime + target_field: aws.cloudtrail.digest.end_time ignore_failure: true formats: - ISO8601 - rename: - field: "json.digestS3Bucket" - target_field: "aws.cloudtrail.digest.s3_bucket" + field: json.digestS3Bucket + target_field: aws.cloudtrail.digest.s3_bucket ignore_failure: true - date: - field: "json.newestEventTime" - target_field: "aws.cloudtrail.digest.newest_event_time" + field: json.newestEventTime + target_field: aws.cloudtrail.digest.newest_event_time ignore_failure: true formats: - ISO8601 - date: - field: "json.oldestEventTime" - target_field: "aws.cloudtrail.digest.oldest_event_time" + field: json.oldestEventTime + target_field: aws.cloudtrail.digest.oldest_event_time ignore_failure: true formats: - ISO8601 - rename: - field: "json.previousDigestS3Bucket" - target_field: "aws.cloudtrail.digest.previous_s3_bucket" + field: json.previousDigestS3Bucket + target_field: aws.cloudtrail.digest.previous_s3_bucket ignore_failure: true - rename: - field: "json.previousDigestHashAlgorithm" - target_field: "aws.cloudtrail.digest.previous_hash_algorithm" + field: json.previousDigestHashAlgorithm + target_field: aws.cloudtrail.digest.previous_hash_algorithm ignore_failure: true - rename: - field: "json.publicKeyFingerprint" - target_field: "aws.cloudtrail.digest.public_key_fingerprint" + field: json.publicKeyFingerprint + target_field: aws.cloudtrail.digest.public_key_fingerprint ignore_failure: true - rename: - field: "json.digestSignatureAlgorithm" - target_field: "aws.cloudtrail.digest.signature_algorithm" + field: json.digestSignatureAlgorithm + target_field: aws.cloudtrail.digest.signature_algorithm ignore_failure: true - rename: - field: "json.insightDetails" - target_field: "aws.cloudtrail.insight_details" + field: json.insightDetails + target_field: aws.cloudtrail.insight_details ignore_failure: true - set: field: group.id @@ -731,7 +739,13 @@ processors: - remove: field: json ignore_missing: true + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - set: - field: "error.message" - value: "{{ _ingest.on_failure_message }}" + field: error.message + value: |- + Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" diff --git a/test/packages/aws/data_stream/cloudtrail/elasticsearch/ingest_pipeline/third-party.yml b/test/packages/aws/data_stream/cloudtrail/elasticsearch/ingest_pipeline/third-party.yml new file mode 100644 index 0000000000..4fc9012b3a --- /dev/null +++ b/test/packages/aws/data_stream/cloudtrail/elasticsearch/ingest_pipeline/third-party.yml @@ -0,0 +1,32 @@ +--- +description: Pipeline for parsing CloudTrail logs from third party api +processors: + - drop: + if: ctx?.json?._raw == null + description: JSON doesn't have CloudTrail data + - fingerprint: + fields: + - _temp_.result._cd + - _temp_.result._indextime + - _temp_.result._raw + - _temp_.result._time + target_field: '_id' + ignore_missing: true + - set: + field: event.original + copy_from: json.result._raw + ignore_empty_value: true + - remove: + field: json + ignore_missing: true + - json: + field: event.original + target_field: json +on_failure: + - append: + field: error.message + value: >- + error in third party api pipeline: + error in [{{_ingest.on_failure_processor_type}}] processor{{#_ingest.on_failure_processor_tag}} + with tag [{{_ingest.on_failure_processor_tag }}]{{/_ingest.on_failure_processor_tag}} + {{ _ingest.on_failure_message }} diff --git a/test/packages/aws/data_stream/cloudtrail/fields/base-fields.yml b/test/packages/aws/data_stream/cloudtrail/fields/base-fields.yml index 7c798f4534..436bcaec7b 100644 --- a/test/packages/aws/data_stream/cloudtrail/fields/base-fields.yml +++ b/test/packages/aws/data_stream/cloudtrail/fields/base-fields.yml @@ -10,3 +10,11 @@ - name: '@timestamp' type: date description: Event timestamp. +- name: event.module + type: constant_keyword + description: Event module + value: aws +- name: event.dataset + type: constant_keyword + description: Event dataset + value: aws.cloudtrail diff --git a/test/packages/aws/data_stream/cloudtrail/fields/ecs.yml b/test/packages/aws/data_stream/cloudtrail/fields/ecs.yml index 81ba6c50b9..f420f22b6b 100644 --- a/test/packages/aws/data_stream/cloudtrail/fields/ecs.yml +++ b/test/packages/aws/data_stream/cloudtrail/fields/ecs.yml @@ -1,150 +1,90 @@ -- name: error.message - type: text - description: Error message. -- name: event.action - type: keyword - description: The action captured by the event. -- name: event.ingested - type: date - description: Timestamp when an event arrived in the central data store. -- name: event.original - type: keyword - description: Raw text message of entire event. Used to demonstrate log integrity. -- name: user.name - type: keyword - description: Short name or login of the user. -- name: user.id - type: keyword - description: Unique identifier of the user. -- name: user.target.name - type: keyword - description: Short name or login of the user. -- name: user.target.id - type: keyword - description: Unique identifier of the user. -- name: user.changes.name - type: keyword - description: Short name or login of the user. -- name: group.id - type: keyword - description: Unique identifier for the group on the system/platform. -- name: group.name - type: keyword - description: Name of the group. -- name: file - title: File - type: group - fields: - - name: path - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Full path to the file, including the file name. It should include the drive letter, when appropriate. - - name: hash.md5 - type: keyword - ignore_above: 1024 - description: MD5 hash. - - name: hash.sha1 - type: keyword - ignore_above: 1024 - description: SHA1 hash. - - name: hash.sha256 - type: keyword - ignore_above: 1024 - description: SHA256 hash. - - name: hash.sha512 - type: keyword - ignore_above: 1024 - description: SHA512 hash. -- name: cloud.account.id - type: keyword - description: The cloud account or organization id used to identify different entities in a multi-tenant environment. -- name: event.provider - type: keyword - description: Source of the event. -- name: cloud.region - type: keyword - description: Region in which this host is running. -- name: source.address - type: keyword - description: Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the .address field. -- name: source.ip - type: ip - description: IP address of the source (IPv4 or IPv6). -- name: user_agent.device.name - type: keyword - description: Name of the device. -- name: user_agent.name - type: keyword - description: Name of the user agent. -- name: user_agent.original - type: keyword - description: Unparsed user_agent string. -- name: user_agent.os.full - type: keyword - description: Operating system name, including the version or code name. -- name: user_agent.os.name - type: keyword - description: Operating system name, without the version. -- name: user_agent.os.version - type: keyword - description: Operating system version as a raw string. -- name: user_agent.version - type: keyword - description: Version of the user agent. -- name: related.user - type: keyword - description: All the user names seen on your event. -- name: related.hash - type: keyword - description: All the hashes seen on your event. -- name: event.kind - type: keyword - description: Event kind (e.g. event, alert, metric, state, pipeline_error, signal) -- name: event.type - type: keyword - description: Event severity (e.g. info, error) -- name: source.as.number - type: long - description: >- - Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. -- name: source.as.organization.name - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Organization name. -- name: source.geo.city_name - type: keyword - ignore_above: 1024 - description: City name. -- name: source.geo.continent_name - type: keyword - ignore_above: 1024 - description: Name of the continent. -- name: source.geo.country_iso_code - type: keyword - ignore_above: 1024 - description: Country ISO code. -- name: source.geo.country_name - type: keyword - ignore_above: 1024 - description: Country name. -- name: source.geo.location +- external: ecs + name: cloud.account.id +- external: ecs + name: cloud.region +- external: ecs + name: ecs.version +- external: ecs + name: error.message +- external: ecs + name: event.action +- external: ecs + name: event.created +- external: ecs + name: event.ingested +- external: ecs + name: event.kind +- external: ecs + name: event.original +- external: ecs + name: event.provider +- external: ecs + name: event.type +- external: ecs + name: file.hash.md5 +- external: ecs + name: file.hash.sha1 +- external: ecs + name: file.hash.sha256 +- external: ecs + name: file.hash.sha512 +- external: ecs + name: file.path +- external: ecs + name: group.id +- external: ecs + name: group.name +- external: ecs + name: related.hash +- external: ecs + name: related.user +- external: ecs + name: source.address +- external: ecs + name: source.as.number +- external: ecs + name: source.as.organization.name +- external: ecs + name: source.geo.city_name +- external: ecs + name: source.geo.continent_name +- external: ecs + name: source.geo.country_iso_code +- external: ecs + name: source.geo.country_name +- description: Longitude and latitude. + level: core + name: source.geo.location type: geo_point - description: Longitude and latitude. -- name: source.geo.region_iso_code - type: keyword - ignore_above: 1024 - description: Region ISO code. -- name: source.geo.region_name - type: keyword - ignore_above: 1024 - description: Region name. +- external: ecs + name: source.geo.region_iso_code +- external: ecs + name: source.geo.region_name +- external: ecs + name: source.ip +- external: ecs + name: tags +- external: ecs + name: user.changes.name +- external: ecs + name: user.id +- external: ecs + name: user.name +- external: ecs + name: user.target.id +- external: ecs + name: user.target.name +- external: ecs + name: user_agent.device.name +- external: ecs + name: user_agent.name +- external: ecs + name: user_agent.original +- external: ecs + name: user_agent.os.full +- external: ecs + name: user_agent.os.name +- external: ecs + name: user_agent.os.version +- external: ecs + name: user_agent.version diff --git a/test/packages/aws/data_stream/cloudtrail/manifest.yml b/test/packages/aws/data_stream/cloudtrail/manifest.yml index 94b8c1d0e3..5db302b0f8 100644 --- a/test/packages/aws/data_stream/cloudtrail/manifest.yml +++ b/test/packages/aws/data_stream/cloudtrail/manifest.yml @@ -1,12 +1,25 @@ -title: AWS CloudTrail logs -release: beta +title: AWS CloudTrail Logs type: logs streams: - - input: s3 - template_path: s3.yml.hbs - title: AWS CloudTrail logs + - input: aws-s3 + template_path: aws-s3.yml.hbs + title: AWS CloudTrail Logs description: Collect AWS CloudTrail logs using s3 input vars: + - name: visibility_timeout + type: text + title: Visibility Timeout + multi: false + required: false + show_user: false + description: The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours. + - name: api_timeout + type: text + title: API Timeout + multi: false + required: false + show_user: false + description: The maximum duration of AWS API can take. The maximum is half of the visibility timeout value. - name: queue_url type: text title: Queue URL @@ -22,3 +35,161 @@ streams: required: false show_user: false description: Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded + - aws-cloudtrail + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: cloudtrail_regex + type: text + title: CloudTrail Logs regex + default: '/CloudTrail/' + required: false + show_user: false + description: | + Regex to match path of CloudTrail S3 Objects. If blank + CloudTrail logs will be skipped. + - name: cloudtrail_digest_regex + type: text + title: CloudTrail Digest Logs regex + default: '/CloudTrail-Digest/' + required: false + show_user: false + description: | + Regex to match path of CloudTrail Digest S3 Objects. If + blank CloudTrail Digest logs will be skipped. + - name: cloudtrail_insight_regex + type: text + title: CloudTrail Insight Logs regex + default: '/CloudTrail-Insight/' + required: false + show_user: false + description: | + Regex to match path of CloudTrail Insight S3 Objects. If + blank CloudTrail Insight logs will be skipped. + - name: max_number_of_messages + type: integer + title: Maximum Concurrent SQS Messages + description: The maximum number of SQS messages that can be inflight at any time. + default: 5 + required: false + show_user: false + - input: httpjson + title: AWS CloudTrail Logs via Splunk Enterprise REST API + description: Collect AWS CloudTrail logs via Splunk Enterprise REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: url + type: text + title: URL of Splunk Enterprise Server + description: i.e. scheme://host:port, path is automatic + show_user: true + required: true + default: https://server.example.com:8089 + - name: username + type: text + title: Splunk REST API Username + show_user: true + required: false + - name: password + type: password + title: Splunk REST API Password + required: false + show_user: true + - name: token + type: password + title: Splunk Authorization Token + description: | + Bearer Token or Session Key, e.g. "Bearer eyJFd3e46..." + or "Splunk 192fd3e...". Cannot be used with username + and password. + show_user: true + required: false + - name: ssl + type: yaml + title: SSL Configuration + multi: false + required: false + show_user: false + description: i.e. certificate_authorities, supported_protocols, verification_mode etc. + default: | + #certificate_authorities: + # - | + # -----BEGIN CERTIFICATE----- + # MIIDCjCCAfKgAwIBAgITJ706Mu2wJlKckpIvkWxEHvEyijANBgkqhkiG9w0BAQsF + # ADAUMRIwEAYDVQQDDAlsb2NhbGhvc3QwIBcNMTkwNzIyMTkyOTA0WhgPMjExOTA2 + # MjgxOTI5MDRaMBQxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEB + # BQADggEPADCCAQoCggEBANce58Y/JykI58iyOXpxGfw0/gMvF0hUQAcUrSMxEO6n + # fZRA49b4OV4SwWmA3395uL2eB2NB8y8qdQ9muXUdPBWE4l9rMZ6gmfu90N5B5uEl + # 94NcfBfYOKi1fJQ9i7WKhTjlRkMCgBkWPkUokvBZFRt8RtF7zI77BSEorHGQCk9t + # /D7BS0GJyfVEhftbWcFEAG3VRcoMhF7kUzYwp+qESoriFRYLeDWv68ZOvG7eoWnP + # PsvZStEVEimjvK5NSESEQa9xWyJOmlOKXhkdymtcUd/nXnx6UTCFgnkgzSdTWV41 + # CI6B6aJ9svCTI2QuoIq2HxX/ix7OvW1huVmcyHVxyUECAwEAAaNTMFEwHQYDVR0O + # BBYEFPwN1OceFGm9v6ux8G+DZ3TUDYxqMB8GA1UdIwQYMBaAFPwN1OceFGm9v6ux + # 8G+DZ3TUDYxqMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAG5D + # 874A4YI7YUwOVsVAdbWtgp1d0zKcPRR+r2OdSbTAV5/gcS3jgBJ3i1BN34JuDVFw + # 3DeJSYT3nxy2Y56lLnxDeF8CUTUtVQx3CuGkRg1ouGAHpO/6OqOhwLLorEmxi7tA + # H2O8mtT0poX5AnOAhzVy7QW0D/k4WaoLyckM5hUa6RtvgvLxOwA0U+VGurCDoctu + # 8F4QOgTAWyh8EZIwaKCliFRSynDpv3JTUwtfZkxo6K6nce1RhCWFAsMvDZL8Dgc0 + # yvgJ38BRsFOtkRuAGSf6ZUwTO8JJRRIFnpUzXflAnGivK9M13D5GEQMmIl6U9Pvk + # sxSmbIUfc2SGJGCJD4I= + # -----END CERTIFICATE----- + - name: interval + type: text + title: Interval to query Splunk Enterprise REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Splunk search string + show_user: true + required: true + default: "search sourcetype=aws:cloudtrail" + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded + - aws-cloudtrail + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false diff --git a/test/packages/aws/data_stream/cloudtrail/sample_event.json b/test/packages/aws/data_stream/cloudtrail/sample_event.json new file mode 100644 index 0000000000..0c8df40b3d --- /dev/null +++ b/test/packages/aws/data_stream/cloudtrail/sample_event.json @@ -0,0 +1,85 @@ +{ + "data_stream": { + "namespace": "default", + "type": "logs", + "dataset": "aws.cloudtrail" + }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "tags": [ + "preserve_original_event" + ], + "cloud": { + "region": "us-east-1", + "account": { + "id": "123456789012" + } + }, + "@timestamp": "2020-01-08T20:53:12.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "user": [ + "Alice", + "Bob", + "Robert" + ] + }, + "event": { + "ingested": "2021-10-05T23:06:12.229540200Z", + "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EX_PRINCIPAL_ID\",\"arn\":\"arn:aws:iam::123456789012:user/Alice\",\"accountId\":\"123456789012\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Alice\"},\"eventTime\":\"2020-01-08T20:53:12Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"UpdateUser\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46\",\"requestParameters\":{\"userName\":\"Bob\",\"newUserName\":\"Robert\"},\"responseElements\":null,\"requestID\":\"3a6b3260-739d-465e-9406-bcEXAMPLE\",\"eventID\":\"9150d546-3564-4262-8e62-110EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"123456789012\"}", + "provider": "iam.amazonaws.com", + "created": "2020-01-08T20:53:12.000Z", + "kind": "event", + "action": "UpdateUser", + "id": "9150d546-3564-4262-8e62-110EXAMPLE", + "type": [ + "user", + "change" + ], + "category": [ + "iam" + ], + "outcome": "success" + }, + "aws": { + "cloudtrail": { + "event_version": "1.05", + "flattened": { + "request_parameters": { + "userName": "Bob", + "newUserName": "Robert" + } + }, + "user_identity": { + "access_key_id": "EXAMPLE_KEY_ID", + "type": "IAMUser", + "arn": "arn:aws:iam::123456789012:user/Alice" + }, + "event_type": "AwsApiCall", + "recipient_account_id": "123456789012", + "request_parameters": "{newUserName=Robert, userName=Bob}" + } + }, + "user": { + "name": "Alice", + "changes": { + "name": "Robert" + }, + "id": "EX_PRINCIPAL_ID", + "target": { + "name": "Bob" + } + }, + "user_agent": { + "name": "aws-cli", + "original": "aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46", + "device": { + "name": "Spider" + }, + "version": "1.16.310" + } +} \ No newline at end of file diff --git a/test/packages/aws/data_stream/cloudwatch_logs/_dev/test/pipeline/test-cloudwatch-ec2.log b/test/packages/aws/data_stream/cloudwatch_logs/_dev/test/pipeline/test-cloudwatch-ec2.log new file mode 100644 index 0000000000..4487fdf08d --- /dev/null +++ b/test/packages/aws/data_stream/cloudwatch_logs/_dev/test/pipeline/test-cloudwatch-ec2.log @@ -0,0 +1,6 @@ +2020-02-20T07:01:01.000Z Feb 20 07:01:01 ip-172-31-81-156 systemd: Stopping User Slice of root. +2020-02-20T07:02:18.000Z Feb 20 07:02:18 ip-172-31-81-156 dhclient[3000]: XMT: Solicit on eth0, interval 125240ms. +2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 dhclient[2898]: DHCPREQUEST on eth0 to 172.31.80.1 port 67 (xid=0x4575af22) +2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 dhclient[2898]: DHCPACK from 172.31.80.1 (xid=0x4575af22) +2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 dhclient[2898]: bound to 172.31.81.156 -- renewal in 1599 seconds. +2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 ec2net: [get_meta] Trying to get http://169.254.169.254/latest/meta-data/network/interfaces/macs/12:e2:a9:95:8b:97/local-ipv4s diff --git a/test/packages/aws/data_stream/cloudwatch_logs/_dev/test/pipeline/test-cloudwatch-ec2.log-expected.json b/test/packages/aws/data_stream/cloudwatch_logs/_dev/test/pipeline/test-cloudwatch-ec2.log-expected.json new file mode 100644 index 0000000000..8e4cce3a5f --- /dev/null +++ b/test/packages/aws/data_stream/cloudwatch_logs/_dev/test/pipeline/test-cloudwatch-ec2.log-expected.json @@ -0,0 +1,112 @@ +{ + "expected": [ + { + "@timestamp": "2020-02-20T07:01:01.000Z", + "ecs": { + "version": "1.12.0" + }, + "event": { + "ingested": "2021-12-09T16:11:58.525004600Z", + "original": "2020-02-20T07:01:01.000Z Feb 20 07:01:01 ip-172-31-81-156 systemd: Stopping User Slice of root." + }, + "aws": { + "cloudwatch": { + "message": "ip-172-31-81-156 systemd: Stopping User Slice of root." + } + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2020-02-20T07:02:18.000Z", + "ecs": { + "version": "1.12.0" + }, + "event": { + "ingested": "2021-12-09T16:11:58.525012700Z", + "original": "2020-02-20T07:02:18.000Z Feb 20 07:02:18 ip-172-31-81-156 dhclient[3000]: XMT: Solicit on eth0, interval 125240ms." + }, + "aws": { + "cloudwatch": { + "message": "ip-172-31-81-156 dhclient[3000]: XMT: Solicit on eth0, interval 125240ms." + } + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2020-02-20T07:02:37.000Z", + "ecs": { + "version": "1.12.0" + }, + "event": { + "ingested": "2021-12-09T16:11:58.525017900Z", + "original": "2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 dhclient[2898]: DHCPREQUEST on eth0 to 172.31.80.1 port 67 (xid=0x4575af22)" + }, + "aws": { + "cloudwatch": { + "message": "ip-172-31-81-156 dhclient[2898]: DHCPREQUEST on eth0 to 172.31.80.1 port 67 (xid=0x4575af22)" + } + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2020-02-20T07:02:37.000Z", + "ecs": { + "version": "1.12.0" + }, + "event": { + "ingested": "2021-12-09T16:11:58.525022500Z", + "original": "2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 dhclient[2898]: DHCPACK from 172.31.80.1 (xid=0x4575af22)" + }, + "aws": { + "cloudwatch": { + "message": "ip-172-31-81-156 dhclient[2898]: DHCPACK from 172.31.80.1 (xid=0x4575af22)" + } + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2020-02-20T07:02:37.000Z", + "ecs": { + "version": "1.12.0" + }, + "event": { + "ingested": "2021-12-09T16:11:58.525027400Z", + "original": "2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 dhclient[2898]: bound to 172.31.81.156 -- renewal in 1599 seconds." + }, + "aws": { + "cloudwatch": { + "message": "ip-172-31-81-156 dhclient[2898]: bound to 172.31.81.156 -- renewal in 1599 seconds." + } + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2020-02-20T07:02:37.000Z", + "ecs": { + "version": "1.12.0" + }, + "event": { + "ingested": "2021-12-09T16:11:58.525032300Z", + "original": "2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 ec2net: [get_meta] Trying to get http://169.254.169.254/latest/meta-data/network/interfaces/macs/12:e2:a9:95:8b:97/local-ipv4s" + }, + "aws": { + "cloudwatch": { + "message": "ip-172-31-81-156 ec2net: [get_meta] Trying to get http://169.254.169.254/latest/meta-data/network/interfaces/macs/12:e2:a9:95:8b:97/local-ipv4s" + } + }, + "tags": [ + "preserve_original_event" + ] + } + ] +} \ No newline at end of file diff --git a/test/packages/aws/data_stream/cloudwatch_logs/_dev/test/pipeline/test-common-config.yml b/test/packages/aws/data_stream/cloudwatch_logs/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 0000000000..5622947e4b --- /dev/null +++ b/test/packages/aws/data_stream/cloudwatch_logs/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,5 @@ +dynamic_fields: + event.ingested: ".*" +fields: + tags: + - preserve_original_event diff --git a/test/packages/aws/data_stream/cloudtrail/agent/stream/s3.yml.hbs b/test/packages/aws/data_stream/cloudwatch_logs/agent/stream/aws-s3.yml.hbs similarity index 64% rename from test/packages/aws/data_stream/cloudtrail/agent/stream/s3.yml.hbs rename to test/packages/aws/data_stream/cloudwatch_logs/agent/stream/aws-s3.yml.hbs index 309c650769..ccf43bcddc 100644 --- a/test/packages/aws/data_stream/cloudtrail/agent/stream/s3.yml.hbs +++ b/test/packages/aws/data_stream/cloudwatch_logs/agent/stream/aws-s3.yml.hbs @@ -1,5 +1,4 @@ queue_url: {{queue_url}} -expand_event_list_from_field: Records {{#if credential_profile_name}} credential_profile_name: {{credential_profile_name}} {{/if}} @@ -12,6 +11,9 @@ visibility_timeout: {{visibility_timeout}} {{#if api_timeout}} api_timeout: {{api_timeout}} {{/if}} +{{#if max_number_of_messages}} +max_number_of_messages: {{max_number_of_messages}} +{{/if}} {{#if endpoint}} endpoint: {{endpoint}} {{/if}} @@ -30,8 +32,20 @@ role_arn: {{role_arn}} {{#if fips_enabled}} fips_enabled: {{fips_enabled}} {{/if}} +{{#if proxy_url }} +proxy_url: {{proxy_url}} +{{/if}} +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} processors: - - add_fields: - target: '' - fields: - ecs.version: 1.8.0 \ No newline at end of file +{{processors}} +{{/if}} \ No newline at end of file diff --git a/test/packages/aws/data_stream/cloudwatch_logs/elasticsearch/ingest_pipeline/default.yml b/test/packages/aws/data_stream/cloudwatch_logs/elasticsearch/ingest_pipeline/default.yml index 1f7317d6dc..af65d9c1a6 100644 --- a/test/packages/aws/data_stream/cloudwatch_logs/elasticsearch/ingest_pipeline/default.yml +++ b/test/packages/aws/data_stream/cloudwatch_logs/elasticsearch/ingest_pipeline/default.yml @@ -2,25 +2,37 @@ description: "Pipeline for CloudWatch logs" processors: - - grok: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + - set: + field: ecs.version + value: '1.12.0' + - rename: field: message + target_field: event.original + ignore_missing: true + - grok: + field: event.original patterns: - - "%{TIMESTAMP_ISO8601:_tmp.timestamp} %{SYSLOGTIMESTAMP:_tmp.syslog_timestamp} %{GREEDYDATA:aws.cloudwatch.message}" - - "%{TIMESTAMP_ISO8601:_tmp.timestamp} %{GREEDYDATA:aws.cloudwatch.message}" - + - '%{TIMESTAMP_ISO8601:_tmp.timestamp} %{SYSLOGTIMESTAMP:_tmp.syslog_timestamp} %{GREEDYDATA:aws.cloudwatch.message}' + - '%{TIMESTAMP_ISO8601:_tmp.timestamp} %{GREEDYDATA:aws.cloudwatch.message}' - date: - field: '_tmp.timestamp' + field: _tmp.timestamp target_field: "@timestamp" ignore_failure: true formats: - - 'ISO8601' - + - ISO8601 - remove: field: - _tmp ignore_missing: true - + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - set: - field: "error.message" - value: "{{ _ingest.on_failure_message }}" + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/test/packages/aws/data_stream/cloudwatch_logs/fields/base-fields.yml b/test/packages/aws/data_stream/cloudwatch_logs/fields/base-fields.yml index 7c798f4534..4fbeaa06a9 100644 --- a/test/packages/aws/data_stream/cloudwatch_logs/fields/base-fields.yml +++ b/test/packages/aws/data_stream/cloudwatch_logs/fields/base-fields.yml @@ -10,3 +10,11 @@ - name: '@timestamp' type: date description: Event timestamp. +- name: event.module + type: constant_keyword + description: Event module + value: aws +- name: event.dataset + type: constant_keyword + description: Event dataset + value: aws.cloudwatch_logs diff --git a/test/packages/aws/data_stream/cloudwatch_logs/fields/ecs.yml b/test/packages/aws/data_stream/cloudwatch_logs/fields/ecs.yml new file mode 100644 index 0000000000..def0bf767f --- /dev/null +++ b/test/packages/aws/data_stream/cloudwatch_logs/fields/ecs.yml @@ -0,0 +1,6 @@ +- external: ecs + name: ecs.version +- external: ecs + name: error.message +- external: ecs + name: tags diff --git a/test/packages/aws/data_stream/cloudwatch_logs/manifest.yml b/test/packages/aws/data_stream/cloudwatch_logs/manifest.yml index 6fc61a757b..7608cf4037 100644 --- a/test/packages/aws/data_stream/cloudwatch_logs/manifest.yml +++ b/test/packages/aws/data_stream/cloudwatch_logs/manifest.yml @@ -1,12 +1,25 @@ title: AWS CloudWatch logs -release: beta type: logs streams: - - input: s3 - template_path: s3.yml.hbs + - input: aws-s3 + template_path: aws-s3.yml.hbs title: AWS CloudWatch logs description: Collect AWS CloudWatch logs using s3 input vars: + - name: visibility_timeout + type: text + title: Visibility Timeout + multi: false + required: false + show_user: false + description: The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours. + - name: api_timeout + type: text + title: API Timeout + multi: false + required: false + show_user: false + description: The maximum duration of AWS API can take. The maximum is half of the visibility timeout value. - name: queue_url type: text title: Queue URL @@ -22,3 +35,36 @@ streams: required: false show_user: false description: Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - aws-cloudwatch-logs + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: max_number_of_messages + type: integer + title: Maximum Concurrent SQS Messages + description: The maximum number of SQS messages that can be inflight at any time. + default: 5 + required: false + show_user: false diff --git a/test/packages/aws/data_stream/cloudwatch_logs/sample_event.json b/test/packages/aws/data_stream/cloudwatch_logs/sample_event.json new file mode 100644 index 0000000000..b41878aaf2 --- /dev/null +++ b/test/packages/aws/data_stream/cloudwatch_logs/sample_event.json @@ -0,0 +1,23 @@ +{ + "@timestamp": "2020-02-20T07:02:37.000Z", + "data_stream": { + "namespace": "default", + "type": "logs", + "dataset": "aws.cloudwatch_logs" + }, + "ecs": { + "version": "1.12.0" + }, + "event": { + "ingested": "2021-07-19T21:47:04.696803300Z", + "original": "2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 ec2net: [get_meta] Trying to get http://169.254.169.254/latest/meta-data/network/interfaces/macs/12:e2:a9:95:8b:97/local-ipv4s" + }, + "aws": { + "cloudwatch": { + "message": "ip-172-31-81-156 ec2net: [get_meta] Trying to get http://169.254.169.254/latest/meta-data/network/interfaces/macs/12:e2:a9:95:8b:97/local-ipv4s" + } + }, + "tags": [ + "preserve_original_event" + ] +} \ No newline at end of file diff --git a/test/packages/aws/data_stream/cloudwatch_metrics/agent/stream/stream.yml.hbs b/test/packages/aws/data_stream/cloudwatch_metrics/agent/stream/stream.yml.hbs index 40e4c2530b..71d53c29ac 100644 --- a/test/packages/aws/data_stream/cloudwatch_metrics/agent/stream/stream.yml.hbs +++ b/test/packages/aws/data_stream/cloudwatch_metrics/agent/stream/stream.yml.hbs @@ -30,3 +30,6 @@ latency: {{latency}} {{#if metrics}} metrics: {{metrics}} {{/if}} +{{#if proxy_url }} +proxy_url: {{proxy_url}} +{{/if}} diff --git a/test/packages/aws/data_stream/cloudwatch_metrics/fields/base-fields.yml b/test/packages/aws/data_stream/cloudwatch_metrics/fields/base-fields.yml index 7c798f4534..901d85d431 100644 --- a/test/packages/aws/data_stream/cloudwatch_metrics/fields/base-fields.yml +++ b/test/packages/aws/data_stream/cloudwatch_metrics/fields/base-fields.yml @@ -10,3 +10,11 @@ - name: '@timestamp' type: date description: Event timestamp. +- name: event.module + type: constant_keyword + description: Event module + value: aws +- name: event.dataset + type: constant_keyword + description: Event dataset + value: aws.cloudwatch_metrics diff --git a/test/packages/aws/data_stream/cloudwatch_metrics/fields/ecs.yml b/test/packages/aws/data_stream/cloudwatch_metrics/fields/ecs.yml index a02d7269c5..83e3f6f122 100644 --- a/test/packages/aws/data_stream/cloudwatch_metrics/fields/ecs.yml +++ b/test/packages/aws/data_stream/cloudwatch_metrics/fields/ecs.yml @@ -1,53 +1,24 @@ -- name: cloud - title: Cloud - group: 2 - type: group - footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' - fields: - - name: account.id - level: extended - type: keyword - description: |- - The cloud account or organization id used to identify different entities in a multi-tenant environment. - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. - ignore_above: 1024 - - name: account.name - level: extended - type: keyword - description: |- - The cloud account name or alias used to identify different entities in a multi-tenant environment. - Examples: AWS account name, Google Cloud ORG display name. - ignore_above: 1024 - - name: availability_zone - level: extended - type: keyword - description: Availability zone in which this host is running. - ignore_above: 1024 - - name: instance.id - level: extended - type: keyword - description: Instance ID of the host machine. - ignore_above: 1024 - - name: machine.type - level: extended - type: keyword - description: Machine type of the host machine. - ignore_above: 1024 - - name: provider - level: extended - type: keyword - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - ignore_above: 1024 - - name: region - level: extended - type: keyword - description: Region in which this host is running. - ignore_above: 1024 -- name: ecs.version - type: keyword - description: ECS version this event conforms to. - example: 1.0.0 - ignore_above: 1024 -- name: service.type - type: keyword - description: Service type +- external: ecs + name: cloud +- external: ecs + name: cloud.account.id +- external: ecs + name: cloud.account.name +- external: ecs + name: cloud.availability_zone +- external: ecs + name: cloud.instance.id +- external: ecs + name: cloud.machine.type +- external: ecs + name: cloud.provider +- external: ecs + name: cloud.region +- external: ecs + name: ecs.version +- external: ecs + name: error +- external: ecs + name: error.message +- external: ecs + name: service.type diff --git a/test/packages/aws/data_stream/cloudwatch_metrics/manifest.yml b/test/packages/aws/data_stream/cloudwatch_metrics/manifest.yml index dae477ae67..543c24a8c5 100644 --- a/test/packages/aws/data_stream/cloudwatch_metrics/manifest.yml +++ b/test/packages/aws/data_stream/cloudwatch_metrics/manifest.yml @@ -1,5 +1,4 @@ title: AWS CloudWatch metrics -release: beta type: metrics streams: - input: aws/metrics diff --git a/test/packages/aws/data_stream/cloudwatch_metrics/sample_event.json b/test/packages/aws/data_stream/cloudwatch_metrics/sample_event.json index 431705cacd..bf25e887b5 100644 --- a/test/packages/aws/data_stream/cloudwatch_metrics/sample_event.json +++ b/test/packages/aws/data_stream/cloudwatch_metrics/sample_event.json @@ -2,7 +2,7 @@ "@timestamp": "2020-05-28T17:17:02.812Z", "event": { "duration": 14119105951, - "dataset": "aws.cloudwatch", + "dataset": "aws.cloudwatch_metrics", "module": "aws" }, "ecs": { diff --git a/test/packages/aws/data_stream/dynamodb/agent/stream/stream.yml.hbs b/test/packages/aws/data_stream/dynamodb/agent/stream/stream.yml.hbs index 07e4a166ac..f6662fd6ce 100644 --- a/test/packages/aws/data_stream/dynamodb/agent/stream/stream.yml.hbs +++ b/test/packages/aws/data_stream/dynamodb/agent/stream/stream.yml.hbs @@ -29,4 +29,7 @@ latency: {{latency}} {{/if}} {{#if tags_filter}} tags_filter: {{tags_filter}} +{{/if}} +{{#if proxy_url }} +proxy_url: {{proxy_url}} {{/if}} \ No newline at end of file diff --git a/test/packages/aws/data_stream/dynamodb/fields/base-fields.yml b/test/packages/aws/data_stream/dynamodb/fields/base-fields.yml index 7c798f4534..f4dcea38bf 100644 --- a/test/packages/aws/data_stream/dynamodb/fields/base-fields.yml +++ b/test/packages/aws/data_stream/dynamodb/fields/base-fields.yml @@ -10,3 +10,11 @@ - name: '@timestamp' type: date description: Event timestamp. +- name: event.module + type: constant_keyword + description: Event module + value: aws +- name: event.dataset + type: constant_keyword + description: Event dataset + value: aws.dynamodb diff --git a/test/packages/aws/data_stream/dynamodb/fields/ecs.yml b/test/packages/aws/data_stream/dynamodb/fields/ecs.yml index a02d7269c5..83e3f6f122 100644 --- a/test/packages/aws/data_stream/dynamodb/fields/ecs.yml +++ b/test/packages/aws/data_stream/dynamodb/fields/ecs.yml @@ -1,53 +1,24 @@ -- name: cloud - title: Cloud - group: 2 - type: group - footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' - fields: - - name: account.id - level: extended - type: keyword - description: |- - The cloud account or organization id used to identify different entities in a multi-tenant environment. - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. - ignore_above: 1024 - - name: account.name - level: extended - type: keyword - description: |- - The cloud account name or alias used to identify different entities in a multi-tenant environment. - Examples: AWS account name, Google Cloud ORG display name. - ignore_above: 1024 - - name: availability_zone - level: extended - type: keyword - description: Availability zone in which this host is running. - ignore_above: 1024 - - name: instance.id - level: extended - type: keyword - description: Instance ID of the host machine. - ignore_above: 1024 - - name: machine.type - level: extended - type: keyword - description: Machine type of the host machine. - ignore_above: 1024 - - name: provider - level: extended - type: keyword - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - ignore_above: 1024 - - name: region - level: extended - type: keyword - description: Region in which this host is running. - ignore_above: 1024 -- name: ecs.version - type: keyword - description: ECS version this event conforms to. - example: 1.0.0 - ignore_above: 1024 -- name: service.type - type: keyword - description: Service type +- external: ecs + name: cloud +- external: ecs + name: cloud.account.id +- external: ecs + name: cloud.account.name +- external: ecs + name: cloud.availability_zone +- external: ecs + name: cloud.instance.id +- external: ecs + name: cloud.machine.type +- external: ecs + name: cloud.provider +- external: ecs + name: cloud.region +- external: ecs + name: ecs.version +- external: ecs + name: error +- external: ecs + name: error.message +- external: ecs + name: service.type diff --git a/test/packages/aws/data_stream/dynamodb/manifest.yml b/test/packages/aws/data_stream/dynamodb/manifest.yml index 12bec6c6e8..437195faec 100644 --- a/test/packages/aws/data_stream/dynamodb/manifest.yml +++ b/test/packages/aws/data_stream/dynamodb/manifest.yml @@ -1,5 +1,4 @@ title: AWS DynamoDB metrics -release: beta type: metrics streams: - input: aws/metrics diff --git a/test/packages/aws/data_stream/ebs/agent/stream/stream.yml.hbs b/test/packages/aws/data_stream/ebs/agent/stream/stream.yml.hbs index b0d8e145fa..df4b1aaf07 100644 --- a/test/packages/aws/data_stream/ebs/agent/stream/stream.yml.hbs +++ b/test/packages/aws/data_stream/ebs/agent/stream/stream.yml.hbs @@ -29,4 +29,7 @@ latency: {{latency}} {{/if}} {{#if tags_filter}} tags_filter: {{tags_filter}} +{{/if}} +{{#if proxy_url }} +proxy_url: {{proxy_url}} {{/if}} \ No newline at end of file diff --git a/test/packages/aws/data_stream/ebs/fields/base-fields.yml b/test/packages/aws/data_stream/ebs/fields/base-fields.yml index 7c798f4534..85dfe5c907 100644 --- a/test/packages/aws/data_stream/ebs/fields/base-fields.yml +++ b/test/packages/aws/data_stream/ebs/fields/base-fields.yml @@ -10,3 +10,11 @@ - name: '@timestamp' type: date description: Event timestamp. +- name: event.module + type: constant_keyword + description: Event module + value: aws +- name: event.dataset + type: constant_keyword + description: Event dataset + value: aws.ebs diff --git a/test/packages/aws/data_stream/ebs/fields/ecs.yml b/test/packages/aws/data_stream/ebs/fields/ecs.yml index a02d7269c5..83e3f6f122 100644 --- a/test/packages/aws/data_stream/ebs/fields/ecs.yml +++ b/test/packages/aws/data_stream/ebs/fields/ecs.yml @@ -1,53 +1,24 @@ -- name: cloud - title: Cloud - group: 2 - type: group - footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' - fields: - - name: account.id - level: extended - type: keyword - description: |- - The cloud account or organization id used to identify different entities in a multi-tenant environment. - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. - ignore_above: 1024 - - name: account.name - level: extended - type: keyword - description: |- - The cloud account name or alias used to identify different entities in a multi-tenant environment. - Examples: AWS account name, Google Cloud ORG display name. - ignore_above: 1024 - - name: availability_zone - level: extended - type: keyword - description: Availability zone in which this host is running. - ignore_above: 1024 - - name: instance.id - level: extended - type: keyword - description: Instance ID of the host machine. - ignore_above: 1024 - - name: machine.type - level: extended - type: keyword - description: Machine type of the host machine. - ignore_above: 1024 - - name: provider - level: extended - type: keyword - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - ignore_above: 1024 - - name: region - level: extended - type: keyword - description: Region in which this host is running. - ignore_above: 1024 -- name: ecs.version - type: keyword - description: ECS version this event conforms to. - example: 1.0.0 - ignore_above: 1024 -- name: service.type - type: keyword - description: Service type +- external: ecs + name: cloud +- external: ecs + name: cloud.account.id +- external: ecs + name: cloud.account.name +- external: ecs + name: cloud.availability_zone +- external: ecs + name: cloud.instance.id +- external: ecs + name: cloud.machine.type +- external: ecs + name: cloud.provider +- external: ecs + name: cloud.region +- external: ecs + name: ecs.version +- external: ecs + name: error +- external: ecs + name: error.message +- external: ecs + name: service.type diff --git a/test/packages/aws/data_stream/ebs/manifest.yml b/test/packages/aws/data_stream/ebs/manifest.yml index 5d0cce0e85..483fb237a5 100644 --- a/test/packages/aws/data_stream/ebs/manifest.yml +++ b/test/packages/aws/data_stream/ebs/manifest.yml @@ -1,5 +1,4 @@ title: AWS EBS metrics -release: beta type: metrics streams: - input: aws/metrics diff --git a/test/packages/aws/data_stream/ec2_logs/_dev/test/pipeline/test-common-config.yml b/test/packages/aws/data_stream/ec2_logs/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 0000000000..5622947e4b --- /dev/null +++ b/test/packages/aws/data_stream/ec2_logs/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,5 @@ +dynamic_fields: + event.ingested: ".*" +fields: + tags: + - preserve_original_event diff --git a/test/packages/aws/data_stream/ec2_logs/_dev/test/pipeline/test-ec2.log b/test/packages/aws/data_stream/ec2_logs/_dev/test/pipeline/test-ec2.log new file mode 100644 index 0000000000..4487fdf08d --- /dev/null +++ b/test/packages/aws/data_stream/ec2_logs/_dev/test/pipeline/test-ec2.log @@ -0,0 +1,6 @@ +2020-02-20T07:01:01.000Z Feb 20 07:01:01 ip-172-31-81-156 systemd: Stopping User Slice of root. +2020-02-20T07:02:18.000Z Feb 20 07:02:18 ip-172-31-81-156 dhclient[3000]: XMT: Solicit on eth0, interval 125240ms. +2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 dhclient[2898]: DHCPREQUEST on eth0 to 172.31.80.1 port 67 (xid=0x4575af22) +2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 dhclient[2898]: DHCPACK from 172.31.80.1 (xid=0x4575af22) +2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 dhclient[2898]: bound to 172.31.81.156 -- renewal in 1599 seconds. +2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 ec2net: [get_meta] Trying to get http://169.254.169.254/latest/meta-data/network/interfaces/macs/12:e2:a9:95:8b:97/local-ipv4s diff --git a/test/packages/aws/data_stream/ec2_logs/_dev/test/pipeline/test-ec2.log-expected.json b/test/packages/aws/data_stream/ec2_logs/_dev/test/pipeline/test-ec2.log-expected.json new file mode 100644 index 0000000000..4298569cb3 --- /dev/null +++ b/test/packages/aws/data_stream/ec2_logs/_dev/test/pipeline/test-ec2.log-expected.json @@ -0,0 +1,136 @@ +{ + "expected": [ + { + "process": { + "name": "systemd" + }, + "@timestamp": "2020-02-20T07:01:01.000Z", + "ecs": { + "version": "1.12.0" + }, + "event": { + "ingested": "2021-12-09T16:11:58.684169900Z", + "original": "2020-02-20T07:01:01.000Z Feb 20 07:01:01 ip-172-31-81-156 systemd: Stopping User Slice of root." + }, + "aws": { + "ec2": { + "ip_address": "ip-172-31-81-156" + } + }, + "message": "Stopping User Slice of root.", + "tags": [ + "preserve_original_event" + ] + }, + { + "process": { + "name": "dhclient[3000]" + }, + "@timestamp": "2020-02-20T07:02:18.000Z", + "ecs": { + "version": "1.12.0" + }, + "event": { + "ingested": "2021-12-09T16:11:58.684178100Z", + "original": "2020-02-20T07:02:18.000Z Feb 20 07:02:18 ip-172-31-81-156 dhclient[3000]: XMT: Solicit on eth0, interval 125240ms." + }, + "aws": { + "ec2": { + "ip_address": "ip-172-31-81-156" + } + }, + "message": "XMT: Solicit on eth0, interval 125240ms.", + "tags": [ + "preserve_original_event" + ] + }, + { + "process": { + "name": "dhclient[2898]" + }, + "@timestamp": "2020-02-20T07:02:37.000Z", + "ecs": { + "version": "1.12.0" + }, + "event": { + "ingested": "2021-12-09T16:11:58.684183300Z", + "original": "2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 dhclient[2898]: DHCPREQUEST on eth0 to 172.31.80.1 port 67 (xid=0x4575af22)" + }, + "aws": { + "ec2": { + "ip_address": "ip-172-31-81-156" + } + }, + "message": "DHCPREQUEST on eth0 to 172.31.80.1 port 67 (xid=0x4575af22)", + "tags": [ + "preserve_original_event" + ] + }, + { + "process": { + "name": "dhclient[2898]" + }, + "@timestamp": "2020-02-20T07:02:37.000Z", + "ecs": { + "version": "1.12.0" + }, + "event": { + "ingested": "2021-12-09T16:11:58.684188400Z", + "original": "2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 dhclient[2898]: DHCPACK from 172.31.80.1 (xid=0x4575af22)" + }, + "aws": { + "ec2": { + "ip_address": "ip-172-31-81-156" + } + }, + "message": "DHCPACK from 172.31.80.1 (xid=0x4575af22)", + "tags": [ + "preserve_original_event" + ] + }, + { + "process": { + "name": "dhclient[2898]" + }, + "@timestamp": "2020-02-20T07:02:37.000Z", + "ecs": { + "version": "1.12.0" + }, + "event": { + "ingested": "2021-12-09T16:11:58.684193500Z", + "original": "2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 dhclient[2898]: bound to 172.31.81.156 -- renewal in 1599 seconds." + }, + "aws": { + "ec2": { + "ip_address": "ip-172-31-81-156" + } + }, + "message": "bound to 172.31.81.156 -- renewal in 1599 seconds.", + "tags": [ + "preserve_original_event" + ] + }, + { + "process": { + "name": "ec2net" + }, + "@timestamp": "2020-02-20T07:02:37.000Z", + "ecs": { + "version": "1.12.0" + }, + "event": { + "ingested": "2021-12-09T16:11:58.684198500Z", + "original": "2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 ec2net: [get_meta] Trying to get http://169.254.169.254/latest/meta-data/network/interfaces/macs/12:e2:a9:95:8b:97/local-ipv4s" + }, + "aws": { + "ec2": { + "ip_address": "ip-172-31-81-156" + } + }, + "message": "[get_meta] Trying to get http://169.254.169.254/latest/meta-data/network/interfaces/macs/12:e2:a9:95:8b:97/local-ipv4s", + "tags": [ + "preserve_original_event" + ] + } + ] +} \ No newline at end of file diff --git a/test/packages/aws/data_stream/cloudwatch_logs/agent/stream/s3.yml.hbs b/test/packages/aws/data_stream/ec2_logs/agent/stream/aws-s3.yml.hbs similarity index 64% rename from test/packages/aws/data_stream/cloudwatch_logs/agent/stream/s3.yml.hbs rename to test/packages/aws/data_stream/ec2_logs/agent/stream/aws-s3.yml.hbs index 6a00835778..ccf43bcddc 100644 --- a/test/packages/aws/data_stream/cloudwatch_logs/agent/stream/s3.yml.hbs +++ b/test/packages/aws/data_stream/ec2_logs/agent/stream/aws-s3.yml.hbs @@ -11,6 +11,9 @@ visibility_timeout: {{visibility_timeout}} {{#if api_timeout}} api_timeout: {{api_timeout}} {{/if}} +{{#if max_number_of_messages}} +max_number_of_messages: {{max_number_of_messages}} +{{/if}} {{#if endpoint}} endpoint: {{endpoint}} {{/if}} @@ -29,8 +32,20 @@ role_arn: {{role_arn}} {{#if fips_enabled}} fips_enabled: {{fips_enabled}} {{/if}} +{{#if proxy_url }} +proxy_url: {{proxy_url}} +{{/if}} +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} processors: - - add_fields: - target: '' - fields: - ecs.version: 1.5.0 \ No newline at end of file +{{processors}} +{{/if}} \ No newline at end of file diff --git a/test/packages/aws/data_stream/ec2_logs/elasticsearch/ingest_pipeline/default.yml b/test/packages/aws/data_stream/ec2_logs/elasticsearch/ingest_pipeline/default.yml index 002e3d24df..db6732f5e0 100644 --- a/test/packages/aws/data_stream/ec2_logs/elasticsearch/ingest_pipeline/default.yml +++ b/test/packages/aws/data_stream/ec2_logs/elasticsearch/ingest_pipeline/default.yml @@ -2,24 +2,36 @@ description: "Pipeline for EC2 logs in CloudWatch" processors: - - grok: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + - set: + field: ecs.version + value: '1.12.0' + - rename: field: message + target_field: event.original + ignore_missing: true + - grok: + field: event.original patterns: - - "%{TIMESTAMP_ISO8601:_tmp.timestamp} %{SYSLOGTIMESTAMP:_tmp.syslog_timestamp} %{IPORHOST:aws.ec2.ip_address} %{DATA:process.name}(?:\\[%{POSINT:process.pid}\\])?: %{GREEDYDATA:message}" - + - '%{TIMESTAMP_ISO8601:_tmp.timestamp} %{SYSLOGTIMESTAMP:_tmp.syslog_timestamp} %{IPORHOST:aws.ec2.ip_address} %{DATA:process.name}(?:\\[%{POSINT:process.pid}\\])?: %{GREEDYDATA:message}' - date: - field: '_tmp.timestamp' - target_field: "@timestamp" + field: _tmp.timestamp + target_field: '@timestamp' ignore_failure: true formats: - - 'ISO8601' - + - ISO8601 - remove: field: - _tmp ignore_missing: true - + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - set: - field: "error.message" - value: "{{ _ingest.on_failure_message }}" + field: 'error.message' + value: '{{ _ingest.on_failure_message }}' diff --git a/test/packages/aws/data_stream/ec2_logs/fields/base-fields.yml b/test/packages/aws/data_stream/ec2_logs/fields/base-fields.yml index 7c798f4534..1cb7e48820 100644 --- a/test/packages/aws/data_stream/ec2_logs/fields/base-fields.yml +++ b/test/packages/aws/data_stream/ec2_logs/fields/base-fields.yml @@ -10,3 +10,11 @@ - name: '@timestamp' type: date description: Event timestamp. +- name: event.module + type: constant_keyword + description: Event module + value: aws +- name: event.dataset + type: constant_keyword + description: Event dataset + value: aws.ec2_logs diff --git a/test/packages/aws/data_stream/ec2_logs/fields/ecs.yml b/test/packages/aws/data_stream/ec2_logs/fields/ecs.yml new file mode 100644 index 0000000000..b190938377 --- /dev/null +++ b/test/packages/aws/data_stream/ec2_logs/fields/ecs.yml @@ -0,0 +1,8 @@ +- external: ecs + name: ecs.version +- external: ecs + name: error.message +- external: ecs + name: message +- external: ecs + name: tags diff --git a/test/packages/aws/data_stream/ec2_logs/manifest.yml b/test/packages/aws/data_stream/ec2_logs/manifest.yml index aca6fb44b4..3c7e8961cd 100644 --- a/test/packages/aws/data_stream/ec2_logs/manifest.yml +++ b/test/packages/aws/data_stream/ec2_logs/manifest.yml @@ -1,12 +1,25 @@ title: AWS EC2 logs -release: beta type: logs streams: - - input: s3 - template_path: s3.yml.hbs + - input: aws-s3 + template_path: aws-s3.yml.hbs title: AWS EC2 logs description: Collect AWS EC2 logs using s3 input vars: + - name: visibility_timeout + type: text + title: Visibility Timeout + multi: false + required: false + show_user: false + description: The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours. + - name: api_timeout + type: text + title: API Timeout + multi: false + required: false + show_user: false + description: The maximum duration of AWS API can take. The maximum is half of the visibility timeout value. - name: queue_url type: text title: Queue URL @@ -22,3 +35,36 @@ streams: required: false show_user: false description: Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - aws-ec2-logs + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: max_number_of_messages + type: integer + title: Maximum Concurrent SQS Messages + description: The maximum number of SQS messages that can be inflight at any time. + default: 5 + required: false + show_user: false diff --git a/test/packages/aws/data_stream/ec2_logs/sample_event.json b/test/packages/aws/data_stream/ec2_logs/sample_event.json new file mode 100644 index 0000000000..a121694710 --- /dev/null +++ b/test/packages/aws/data_stream/ec2_logs/sample_event.json @@ -0,0 +1,27 @@ +{ + "data_stream": { + "namespace": "default", + "type": "logs", + "dataset": "aws.ec2_logs" + }, + "process": { + "name": "systemd" + }, + "@timestamp": "2020-02-20T07:01:01.000Z", + "ecs": { + "version": "1.12.0" + }, + "event": { + "ingested": "2021-07-19T21:47:04.871450600Z", + "original": "2020-02-20T07:01:01.000Z Feb 20 07:01:01 ip-172-31-81-156 systemd: Stopping User Slice of root." + }, + "aws": { + "ec2": { + "ip_address": "ip-172-31-81-156" + } + }, + "message": "Stopping User Slice of root.", + "tags": [ + "preserve_original_event" + ] +} \ No newline at end of file diff --git a/test/packages/aws/data_stream/ec2_metrics/_dev/test/system/test-default-config.yml b/test/packages/aws/data_stream/ec2_metrics/_dev/test/system/test-default-config.yml index 0f384a1fe5..15d7d57b30 100644 --- a/test/packages/aws/data_stream/ec2_metrics/_dev/test/system/test-default-config.yml +++ b/test/packages/aws/data_stream/ec2_metrics/_dev/test/system/test-default-config.yml @@ -1,4 +1,4 @@ -wait_for_data_timeout: 20m # AWS CloudWatch may delay metrics delivery for more that 10 minutes. +wait_for_data_timeout: 20m # AWS CloudWatch may delay metrics delivery for more than 10 minutes. vars: access_key_id: '{{AWS_ACCESS_KEY_ID}}' secret_access_key: '{{AWS_SECRET_ACCESS_KEY}}' diff --git a/test/packages/aws/data_stream/ec2_metrics/agent/stream/stream.yml.hbs b/test/packages/aws/data_stream/ec2_metrics/agent/stream/stream.yml.hbs index 5eb40ca78b..d1c7ff7dd4 100644 --- a/test/packages/aws/data_stream/ec2_metrics/agent/stream/stream.yml.hbs +++ b/test/packages/aws/data_stream/ec2_metrics/agent/stream/stream.yml.hbs @@ -29,4 +29,7 @@ latency: {{latency}} {{/if}} {{#if tags_filter}} tags_filter: {{tags_filter}} +{{/if}} +{{#if proxy_url }} +proxy_url: {{proxy_url}} {{/if}} \ No newline at end of file diff --git a/test/packages/aws/data_stream/ec2_metrics/fields/base-fields.yml b/test/packages/aws/data_stream/ec2_metrics/fields/base-fields.yml index 7c798f4534..9e545fc4a7 100644 --- a/test/packages/aws/data_stream/ec2_metrics/fields/base-fields.yml +++ b/test/packages/aws/data_stream/ec2_metrics/fields/base-fields.yml @@ -10,3 +10,11 @@ - name: '@timestamp' type: date description: Event timestamp. +- name: event.module + type: constant_keyword + description: Event module + value: aws +- name: event.dataset + type: constant_keyword + description: Event dataset + value: aws.ec2_metrics diff --git a/test/packages/aws/data_stream/ec2_metrics/fields/ecs.yml b/test/packages/aws/data_stream/ec2_metrics/fields/ecs.yml index a02d7269c5..83e3f6f122 100644 --- a/test/packages/aws/data_stream/ec2_metrics/fields/ecs.yml +++ b/test/packages/aws/data_stream/ec2_metrics/fields/ecs.yml @@ -1,53 +1,24 @@ -- name: cloud - title: Cloud - group: 2 - type: group - footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' - fields: - - name: account.id - level: extended - type: keyword - description: |- - The cloud account or organization id used to identify different entities in a multi-tenant environment. - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. - ignore_above: 1024 - - name: account.name - level: extended - type: keyword - description: |- - The cloud account name or alias used to identify different entities in a multi-tenant environment. - Examples: AWS account name, Google Cloud ORG display name. - ignore_above: 1024 - - name: availability_zone - level: extended - type: keyword - description: Availability zone in which this host is running. - ignore_above: 1024 - - name: instance.id - level: extended - type: keyword - description: Instance ID of the host machine. - ignore_above: 1024 - - name: machine.type - level: extended - type: keyword - description: Machine type of the host machine. - ignore_above: 1024 - - name: provider - level: extended - type: keyword - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - ignore_above: 1024 - - name: region - level: extended - type: keyword - description: Region in which this host is running. - ignore_above: 1024 -- name: ecs.version - type: keyword - description: ECS version this event conforms to. - example: 1.0.0 - ignore_above: 1024 -- name: service.type - type: keyword - description: Service type +- external: ecs + name: cloud +- external: ecs + name: cloud.account.id +- external: ecs + name: cloud.account.name +- external: ecs + name: cloud.availability_zone +- external: ecs + name: cloud.instance.id +- external: ecs + name: cloud.machine.type +- external: ecs + name: cloud.provider +- external: ecs + name: cloud.region +- external: ecs + name: ecs.version +- external: ecs + name: error +- external: ecs + name: error.message +- external: ecs + name: service.type diff --git a/test/packages/aws/data_stream/ec2_metrics/manifest.yml b/test/packages/aws/data_stream/ec2_metrics/manifest.yml index 8a3d5fb87f..1a78ff9099 100644 --- a/test/packages/aws/data_stream/ec2_metrics/manifest.yml +++ b/test/packages/aws/data_stream/ec2_metrics/manifest.yml @@ -1,5 +1,4 @@ title: AWS EC2 metrics -release: beta type: metrics streams: - input: aws/metrics diff --git a/test/packages/aws/data_stream/ec2_metrics/sample_event.json b/test/packages/aws/data_stream/ec2_metrics/sample_event.json index ffdd822660..6969b62dbc 100644 --- a/test/packages/aws/data_stream/ec2_metrics/sample_event.json +++ b/test/packages/aws/data_stream/ec2_metrics/sample_event.json @@ -83,7 +83,7 @@ "event": { "module": "aws", "duration": 23217499283, - "dataset": "aws.ec2" + "dataset": "aws.ec2_metrics" }, "metricset": { "period": 300000, diff --git a/test/packages/aws/data_stream/elb_logs/_dev/test/pipeline/test-alb.log-expected.json b/test/packages/aws/data_stream/elb_logs/_dev/test/pipeline/test-alb.log-expected.json index 69935eea78..baf96cc0f7 100644 --- a/test/packages/aws/data_stream/elb_logs/_dev/test/pipeline/test-alb.log-expected.json +++ b/test/packages/aws/data_stream/elb_logs/_dev/test/pipeline/test-alb.log-expected.json @@ -1,22 +1,38 @@ { "expected": [ { - "cloud": { - "provider": "aws" - }, "tracing": { "trace": { "id": "Root=1-58337262-36d228ad5d99923122bbe354" } }, + "source": { + "port": "2817", + "ip": "192.168.131.39" + }, + "url": { + "path": "/", + "original": "http://www.example.com:80/", + "scheme": "http", + "port": 80, + "domain": "www.example.com" + }, + "tags": [ + "preserve_original_event" + ], + "cloud": { + "provider": "aws" + }, "@timestamp": "2018-07-02T22:23:00.186Z", + "ecs": { + "version": "1.12.0" + }, "http": { "request": { "method": "get", "body": { "bytes": 34 - }, - "referrer": "http://www.example.com:80/" + } }, "version": "1.1", "response": { @@ -26,9 +42,14 @@ "status_code": 200 } }, - "source": { - "port": "2817", - "ip": "192.168.131.39" + "event": { + "ingested": "2021-12-09T16:11:58.868846100Z", + "original": "http 2018-07-02T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 192.168.131.39:2817 10.0.0.1:80 0.000 0.001 0.000 200 200 34 366 \"GET http://www.example.com:80/ HTTP/1.1\" \"curl/7.46.0\" - - arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 \"Root=1-58337262-36d228ad5d99923122bbe354\" \"-\" \"-\" 0 2018-07-02T22:22:48.364000Z \"forward,redirect\" \"-\" \"-\" \"10.0.0.1:80\" \"200\" \"-\" \"-\"", + "kind": "event", + "start": "2018-07-02T22:22:48.364000Z", + "end": "2018-07-02T22:23:00.186Z", + "category": "web", + "outcome": "success" }, "aws": { "elb": { @@ -70,15 +91,13 @@ ] } }, - "event": { - "start": "2018-07-02T22:22:48.364000Z", - "end": "2018-07-02T22:23:00.186Z", - "category": "web", - "kind": "event", - "outcome": "success" - }, "user_agent": { - "original": "curl/7.46.0" + "name": "curl", + "original": "curl/7.46.0", + "device": { + "name": "Other" + }, + "version": "7.46.0" } } ] diff --git a/test/packages/aws/data_stream/elb_logs/_dev/test/pipeline/test-common-config.yml b/test/packages/aws/data_stream/elb_logs/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 0000000000..5622947e4b --- /dev/null +++ b/test/packages/aws/data_stream/elb_logs/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,5 @@ +dynamic_fields: + event.ingested: ".*" +fields: + tags: + - preserve_original_event diff --git a/test/packages/aws/data_stream/ec2_logs/agent/stream/s3.yml.hbs b/test/packages/aws/data_stream/elb_logs/agent/stream/aws-s3.yml.hbs similarity index 64% rename from test/packages/aws/data_stream/ec2_logs/agent/stream/s3.yml.hbs rename to test/packages/aws/data_stream/elb_logs/agent/stream/aws-s3.yml.hbs index 6a00835778..ccf43bcddc 100644 --- a/test/packages/aws/data_stream/ec2_logs/agent/stream/s3.yml.hbs +++ b/test/packages/aws/data_stream/elb_logs/agent/stream/aws-s3.yml.hbs @@ -11,6 +11,9 @@ visibility_timeout: {{visibility_timeout}} {{#if api_timeout}} api_timeout: {{api_timeout}} {{/if}} +{{#if max_number_of_messages}} +max_number_of_messages: {{max_number_of_messages}} +{{/if}} {{#if endpoint}} endpoint: {{endpoint}} {{/if}} @@ -29,8 +32,20 @@ role_arn: {{role_arn}} {{#if fips_enabled}} fips_enabled: {{fips_enabled}} {{/if}} +{{#if proxy_url }} +proxy_url: {{proxy_url}} +{{/if}} +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} processors: - - add_fields: - target: '' - fields: - ecs.version: 1.5.0 \ No newline at end of file +{{processors}} +{{/if}} \ No newline at end of file diff --git a/test/packages/aws/data_stream/elb_logs/elasticsearch/ingest_pipeline/default.yml b/test/packages/aws/data_stream/elb_logs/elasticsearch/ingest_pipeline/default.yml index b7edf61ada..10dbec91e2 100644 --- a/test/packages/aws/data_stream/elb_logs/elasticsearch/ingest_pipeline/default.yml +++ b/test/packages/aws/data_stream/elb_logs/elasticsearch/ingest_pipeline/default.yml @@ -2,8 +2,18 @@ description: "Pipeline for ELB logs" processors: - - grok: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + - set: + field: ecs.version + value: '1.12.0' + - rename: field: message + target_field: event.original + ignore_missing: true + - grok: + field: event.original # Classic ELB patterns documented in https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/access-log-collection.html # ELB v2 Application load balancers https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-access-logs.html # ELB v2 Netwwork load balancers https://docs.aws.amazon.com/elasticloadbalancing/latest/network/load-balancer-access-logs.html @@ -75,8 +85,8 @@ processors: (?:-|%{NUMBER:aws.elb.backend.http.response.status_code:long}) %{NUMBER:http.request.body.bytes:long} %{NUMBER:http.response.body.bytes:long} - \"(?:-|%{WORD:http.request.method}) (?:-|%{NOTSPACE:http.request.referrer}) (?:-|HTTP/%{NOTSPACE:http.version})\" - \"%{DATA:user_agent.original}\" + \"(?:-|%{WORD:http.request.method}) (?:-|%{NOTSPACE:_tmp.uri_orig}) (?:-|HTTP/%{NOTSPACE:http.version})\" + \"%{DATA:_tmp.user_agent}\" %{ELBSSL} ELBTCPLOG: >- %{ELBCOMMON} @@ -89,110 +99,100 @@ processors: %{ELBSSL} ELBV2TYPE: '%{WORD:aws.elb.type}' ELBV2LOGVERSION: '%{NOTSPACE}' # Could be used to support different log versions, only 1.0 exists now - - set: field: event.kind value: event - - set: field: cloud.provider value: aws - - set: - if: 'ctx.http != null' - field: 'aws.elb.protocol' - value: 'http' + if: ctx.http != null + field: aws.elb.protocol + value: http + + - uri_parts: + if: 'ctx?._tmp?.uri_orig != null' + field: _tmp.uri_orig + ignore_failure: true + + - user_agent: + if: 'ctx?._tmp?.user_agent != null' + field: _tmp.user_agent + ignore_missing: true - set: - if: 'ctx.http != null' + if: ctx.http != null field: event.category value: web - - set: - if: 'ctx.http == null' - field: 'aws.elb.protocol' - value: 'tcp' - + field: aws.elb.protocol + value: tcp + if: ctx.http == null - set: - if: 'ctx.http == null' field: event.category value: network - + if: ctx.http == null - set: - if: 'ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400' field: event.outcome value: success - + if: 'ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400' - set: - if: 'ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400' field: event.outcome value: failure - + if: 'ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400' - lowercase: field: http.request.method ignore_missing: true - - set: - if: "ctx?.aws?.elb?.trace_id != null" field: tracing.trace.id - value: "{{aws.elb.trace_id}}" - + value: '{{aws.elb.trace_id}}' + if: ctx?.aws?.elb?.trace_id != null - split: - field: '_tmp.actions_executed' - target_field: 'aws.elb.action_executed' + field: _tmp.actions_executed + target_field: aws.elb.action_executed separator: ',' ignore_missing: true - - split: - field: '_tmp.target_port' - target_field: 'aws.elb.target_port' + field: _tmp.target_port + target_field: aws.elb.target_port separator: ' ' ignore_missing: true - - split: - field: '_tmp.target_status_code' - target_field: 'aws.elb.target_status_code' + field: _tmp.target_status_code + target_field: aws.elb.target_status_code separator: ' ' ignore_missing: true - - date: - field: '_tmp.timestamp' + field: _tmp.timestamp formats: - - 'ISO8601' - + - ISO8601 - set: - field: 'event.end' + field: event.end value: '{{ @timestamp }}' - - geoip: - field: 'source.ip' - target_field: 'source.geo' + field: source.ip + target_field: source.geo ignore_missing: true - - geoip: - database_file: 'GeoLite2-ASN.mmdb' - field: 'source.ip' - target_field: 'source.as' + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as properties: - - 'asn' - - 'organization_name' + - asn + - organization_name ignore_missing: true - - rename: field: source.as.asn target_field: source.as.number ignore_missing: true - - rename: field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - - set: field: tls.cipher value: '{{aws.elb.ssl_cipher}}' if: ctx.aws?.elb?.ssl_cipher != null - - script: lang: painless if: ctx.aws?.elb?.ssl_protocol != null @@ -207,14 +207,16 @@ processors: ctx.tls.version = parts[1].substring(0,1) + "." + parts[1].substring(1); } ctx.tls.version_protocol = parts[0].toLowerCase(); - - remove: field: - - message - _tmp ignore_missing: true - + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - set: - field: "error.message" - value: "{{ _ingest.on_failure_message }}" + field: 'error.message' + value: '{{ _ingest.on_failure_message }}' diff --git a/test/packages/aws/data_stream/elb_logs/fields/base-fields.yml b/test/packages/aws/data_stream/elb_logs/fields/base-fields.yml index 7c798f4534..fedbf54e90 100644 --- a/test/packages/aws/data_stream/elb_logs/fields/base-fields.yml +++ b/test/packages/aws/data_stream/elb_logs/fields/base-fields.yml @@ -10,3 +10,11 @@ - name: '@timestamp' type: date description: Event timestamp. +- name: event.module + type: constant_keyword + description: Event module + value: aws +- name: event.dataset + type: constant_keyword + description: Event dataset + value: aws.elb_logs diff --git a/test/packages/aws/data_stream/elb_logs/fields/ecs.yml b/test/packages/aws/data_stream/elb_logs/fields/ecs.yml new file mode 100644 index 0000000000..00a5436510 --- /dev/null +++ b/test/packages/aws/data_stream/elb_logs/fields/ecs.yml @@ -0,0 +1,22 @@ +- external: ecs + name: ecs.version +- external: ecs + name: error.message +- external: ecs + name: tags +- external: ecs + name: url.domain +- external: ecs + name: url.original +- external: ecs + name: url.path +- external: ecs + name: url.port +- external: ecs + name: url.scheme +- external: ecs + name: user_agent.device.name +- external: ecs + name: user_agent.name +- external: ecs + name: user_agent.version diff --git a/test/packages/aws/data_stream/elb_logs/manifest.yml b/test/packages/aws/data_stream/elb_logs/manifest.yml index 4fcba786fc..fdd4f2549e 100644 --- a/test/packages/aws/data_stream/elb_logs/manifest.yml +++ b/test/packages/aws/data_stream/elb_logs/manifest.yml @@ -1,12 +1,25 @@ title: AWS ELB logs -release: beta type: logs streams: - - input: s3 - template_path: s3.yml.hbs + - input: aws-s3 + template_path: aws-s3.yml.hbs title: AWS ELB logs description: Collect AWS ELB logs using s3 input vars: + - name: visibility_timeout + type: text + title: Visibility Timeout + multi: false + required: false + show_user: false + description: The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours. + - name: api_timeout + type: text + title: API Timeout + multi: false + required: false + show_user: false + description: The maximum duration of AWS API can take. The maximum is half of the visibility timeout value. - name: queue_url type: text title: Queue URL @@ -22,3 +35,36 @@ streams: required: false show_user: false description: Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - aws-elb-logs + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: max_number_of_messages + type: integer + title: Maximum Concurrent SQS Messages + description: The maximum number of SQS messages that can be inflight at any time. + default: 5 + required: false + show_user: false diff --git a/test/packages/aws/data_stream/elb_logs/sample_event.json b/test/packages/aws/data_stream/elb_logs/sample_event.json new file mode 100644 index 0000000000..d0d9729d89 --- /dev/null +++ b/test/packages/aws/data_stream/elb_logs/sample_event.json @@ -0,0 +1,105 @@ +{ + "data_stream": { + "namespace": "default", + "type": "logs", + "dataset": "aws.elb_logs" + }, + "tracing": { + "trace": { + "id": "Root=1-58337262-36d228ad5d99923122bbe354" + } + }, + "source": { + "port": "2817", + "ip": "192.168.131.39" + }, + "url": { + "path": "/", + "original": "http://www.example.com:80/", + "scheme": "http", + "port": 80, + "domain": "www.example.com" + }, + "tags": [ + "preserve_original_event" + ], + "cloud": { + "provider": "aws" + }, + "@timestamp": "2018-07-02T22:23:00.186Z", + "ecs": { + "version": "1.12.0" + }, + "http": { + "request": { + "method": "get", + "body": { + "bytes": 34 + } + }, + "version": "1.1", + "response": { + "body": { + "bytes": 366 + }, + "status_code": 200 + } + }, + "event": { + "ingested": "2021-07-19T21:47:05.084930900Z", + "original": "http 2018-07-02T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 192.168.131.39:2817 10.0.0.1:80 0.000 0.001 0.000 200 200 34 366 \"GET http://www.example.com:80/ HTTP/1.1\" \"curl/7.46.0\" - - arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 \"Root=1-58337262-36d228ad5d99923122bbe354\" \"-\" \"-\" 0 2018-07-02T22:22:48.364000Z \"forward,redirect\" \"-\" \"-\" \"10.0.0.1:80\" \"200\" \"-\" \"-\"", + "kind": "event", + "start": "2018-07-02T22:22:48.364000Z", + "end": "2018-07-02T22:23:00.186Z", + "category": "web", + "outcome": "success" + }, + "aws": { + "elb": { + "trace_id": "Root=1-58337262-36d228ad5d99923122bbe354", + "matched_rule_priority": "0", + "type": "http", + "request_processing_time": { + "sec": 0.0 + }, + "response_processing_time": { + "sec": 0.0 + }, + "target_port": [ + "10.0.0.1:80" + ], + "protocol": "http", + "target_status_code": [ + "200" + ], + "name": "app/my-loadbalancer/50dc6c495c0c9188", + "backend": { + "port": "80", + "http": { + "response": { + "status_code": 200 + } + }, + "ip": "10.0.0.1" + }, + "target_group": { + "arn": "arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067" + }, + "backend_processing_time": { + "sec": 0.001 + }, + "action_executed": [ + "forward", + "redirect" + ] + } + }, + "user_agent": { + "name": "curl", + "original": "curl/7.46.0", + "device": { + "name": "Other" + }, + "version": "7.46.0" + } +} \ No newline at end of file diff --git a/test/packages/aws/data_stream/elb_metrics/agent/stream/stream.yml.hbs b/test/packages/aws/data_stream/elb_metrics/agent/stream/stream.yml.hbs index 57c5acdd4c..1fbf0974f1 100644 --- a/test/packages/aws/data_stream/elb_metrics/agent/stream/stream.yml.hbs +++ b/test/packages/aws/data_stream/elb_metrics/agent/stream/stream.yml.hbs @@ -29,4 +29,7 @@ latency: {{latency}} {{/if}} {{#if tags_filter}} tags_filter: {{tags_filter}} +{{/if}} +{{#if proxy_url }} +proxy_url: {{proxy_url}} {{/if}} \ No newline at end of file diff --git a/test/packages/aws/data_stream/elb_metrics/fields/base-fields.yml b/test/packages/aws/data_stream/elb_metrics/fields/base-fields.yml index 7c798f4534..63e855deea 100644 --- a/test/packages/aws/data_stream/elb_metrics/fields/base-fields.yml +++ b/test/packages/aws/data_stream/elb_metrics/fields/base-fields.yml @@ -10,3 +10,11 @@ - name: '@timestamp' type: date description: Event timestamp. +- name: event.module + type: constant_keyword + description: Event module + value: aws +- name: event.dataset + type: constant_keyword + description: Event dataset + value: aws.elb_metrics diff --git a/test/packages/aws/data_stream/elb_metrics/fields/ecs.yml b/test/packages/aws/data_stream/elb_metrics/fields/ecs.yml index a02d7269c5..83e3f6f122 100644 --- a/test/packages/aws/data_stream/elb_metrics/fields/ecs.yml +++ b/test/packages/aws/data_stream/elb_metrics/fields/ecs.yml @@ -1,53 +1,24 @@ -- name: cloud - title: Cloud - group: 2 - type: group - footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' - fields: - - name: account.id - level: extended - type: keyword - description: |- - The cloud account or organization id used to identify different entities in a multi-tenant environment. - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. - ignore_above: 1024 - - name: account.name - level: extended - type: keyword - description: |- - The cloud account name or alias used to identify different entities in a multi-tenant environment. - Examples: AWS account name, Google Cloud ORG display name. - ignore_above: 1024 - - name: availability_zone - level: extended - type: keyword - description: Availability zone in which this host is running. - ignore_above: 1024 - - name: instance.id - level: extended - type: keyword - description: Instance ID of the host machine. - ignore_above: 1024 - - name: machine.type - level: extended - type: keyword - description: Machine type of the host machine. - ignore_above: 1024 - - name: provider - level: extended - type: keyword - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - ignore_above: 1024 - - name: region - level: extended - type: keyword - description: Region in which this host is running. - ignore_above: 1024 -- name: ecs.version - type: keyword - description: ECS version this event conforms to. - example: 1.0.0 - ignore_above: 1024 -- name: service.type - type: keyword - description: Service type +- external: ecs + name: cloud +- external: ecs + name: cloud.account.id +- external: ecs + name: cloud.account.name +- external: ecs + name: cloud.availability_zone +- external: ecs + name: cloud.instance.id +- external: ecs + name: cloud.machine.type +- external: ecs + name: cloud.provider +- external: ecs + name: cloud.region +- external: ecs + name: ecs.version +- external: ecs + name: error +- external: ecs + name: error.message +- external: ecs + name: service.type diff --git a/test/packages/aws/data_stream/elb_metrics/manifest.yml b/test/packages/aws/data_stream/elb_metrics/manifest.yml index 1e6ed4e207..91ea317594 100644 --- a/test/packages/aws/data_stream/elb_metrics/manifest.yml +++ b/test/packages/aws/data_stream/elb_metrics/manifest.yml @@ -1,5 +1,4 @@ title: AWS ELB metrics -release: beta type: metrics streams: - input: aws/metrics diff --git a/test/packages/aws/data_stream/elb_metrics/sample_event.json b/test/packages/aws/data_stream/elb_metrics/sample_event.json index d187909719..a2def82582 100644 --- a/test/packages/aws/data_stream/elb_metrics/sample_event.json +++ b/test/packages/aws/data_stream/elb_metrics/sample_event.json @@ -53,7 +53,7 @@ "period": 60000 }, "event": { - "dataset": "aws.elb", + "dataset": "aws.elb_metrics", "module": "aws", "duration": 15044430616 }, diff --git a/test/packages/aws/data_stream/lambda/agent/stream/stream.yml.hbs b/test/packages/aws/data_stream/lambda/agent/stream/stream.yml.hbs index cf17d23388..0819b829a7 100644 --- a/test/packages/aws/data_stream/lambda/agent/stream/stream.yml.hbs +++ b/test/packages/aws/data_stream/lambda/agent/stream/stream.yml.hbs @@ -29,4 +29,7 @@ latency: {{latency}} {{/if}} {{#if tags_filter}} tags_filter: {{tags_filter}} +{{/if}} +{{#if proxy_url }} +proxy_url: {{proxy_url}} {{/if}} \ No newline at end of file diff --git a/test/packages/aws/data_stream/lambda/fields/base-fields.yml b/test/packages/aws/data_stream/lambda/fields/base-fields.yml index 7c798f4534..07320d3db7 100644 --- a/test/packages/aws/data_stream/lambda/fields/base-fields.yml +++ b/test/packages/aws/data_stream/lambda/fields/base-fields.yml @@ -10,3 +10,11 @@ - name: '@timestamp' type: date description: Event timestamp. +- name: event.module + type: constant_keyword + description: Event module + value: aws +- name: event.dataset + type: constant_keyword + description: Event dataset + value: aws.lambda diff --git a/test/packages/aws/data_stream/lambda/fields/ecs.yml b/test/packages/aws/data_stream/lambda/fields/ecs.yml index a02d7269c5..83e3f6f122 100644 --- a/test/packages/aws/data_stream/lambda/fields/ecs.yml +++ b/test/packages/aws/data_stream/lambda/fields/ecs.yml @@ -1,53 +1,24 @@ -- name: cloud - title: Cloud - group: 2 - type: group - footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' - fields: - - name: account.id - level: extended - type: keyword - description: |- - The cloud account or organization id used to identify different entities in a multi-tenant environment. - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. - ignore_above: 1024 - - name: account.name - level: extended - type: keyword - description: |- - The cloud account name or alias used to identify different entities in a multi-tenant environment. - Examples: AWS account name, Google Cloud ORG display name. - ignore_above: 1024 - - name: availability_zone - level: extended - type: keyword - description: Availability zone in which this host is running. - ignore_above: 1024 - - name: instance.id - level: extended - type: keyword - description: Instance ID of the host machine. - ignore_above: 1024 - - name: machine.type - level: extended - type: keyword - description: Machine type of the host machine. - ignore_above: 1024 - - name: provider - level: extended - type: keyword - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - ignore_above: 1024 - - name: region - level: extended - type: keyword - description: Region in which this host is running. - ignore_above: 1024 -- name: ecs.version - type: keyword - description: ECS version this event conforms to. - example: 1.0.0 - ignore_above: 1024 -- name: service.type - type: keyword - description: Service type +- external: ecs + name: cloud +- external: ecs + name: cloud.account.id +- external: ecs + name: cloud.account.name +- external: ecs + name: cloud.availability_zone +- external: ecs + name: cloud.instance.id +- external: ecs + name: cloud.machine.type +- external: ecs + name: cloud.provider +- external: ecs + name: cloud.region +- external: ecs + name: ecs.version +- external: ecs + name: error +- external: ecs + name: error.message +- external: ecs + name: service.type diff --git a/test/packages/aws/data_stream/lambda/manifest.yml b/test/packages/aws/data_stream/lambda/manifest.yml index 5e0684218b..61505e42ed 100644 --- a/test/packages/aws/data_stream/lambda/manifest.yml +++ b/test/packages/aws/data_stream/lambda/manifest.yml @@ -1,5 +1,4 @@ title: AWS Lambda metrics -release: beta type: metrics streams: - input: aws/metrics diff --git a/test/packages/aws/data_stream/lambda/sample_event.json b/test/packages/aws/data_stream/lambda/sample_event.json index b1542233bd..11d616213b 100644 --- a/test/packages/aws/data_stream/lambda/sample_event.json +++ b/test/packages/aws/data_stream/lambda/sample_event.json @@ -8,7 +8,7 @@ "version": "8.0.0" }, "event": { - "dataset": "aws.dynamodb", + "dataset": "aws.lambda", "module": "aws", "duration": 10266182336 }, diff --git a/test/packages/aws/data_stream/natgateway/agent/stream/stream.yml.hbs b/test/packages/aws/data_stream/natgateway/agent/stream/stream.yml.hbs index 94bed66ae8..23a1ed0cab 100644 --- a/test/packages/aws/data_stream/natgateway/agent/stream/stream.yml.hbs +++ b/test/packages/aws/data_stream/natgateway/agent/stream/stream.yml.hbs @@ -29,4 +29,7 @@ latency: {{latency}} {{/if}} {{#if tags_filter}} tags_filter: {{tags_filter}} +{{/if}} +{{#if proxy_url }} +proxy_url: {{proxy_url}} {{/if}} \ No newline at end of file diff --git a/test/packages/aws/data_stream/natgateway/fields/base-fields.yml b/test/packages/aws/data_stream/natgateway/fields/base-fields.yml index 7c798f4534..436e8fb587 100644 --- a/test/packages/aws/data_stream/natgateway/fields/base-fields.yml +++ b/test/packages/aws/data_stream/natgateway/fields/base-fields.yml @@ -10,3 +10,11 @@ - name: '@timestamp' type: date description: Event timestamp. +- name: event.module + type: constant_keyword + description: Event module + value: aws +- name: event.dataset + type: constant_keyword + description: Event dataset + value: aws.natgateway diff --git a/test/packages/aws/data_stream/natgateway/fields/ecs.yml b/test/packages/aws/data_stream/natgateway/fields/ecs.yml index a02d7269c5..83e3f6f122 100644 --- a/test/packages/aws/data_stream/natgateway/fields/ecs.yml +++ b/test/packages/aws/data_stream/natgateway/fields/ecs.yml @@ -1,53 +1,24 @@ -- name: cloud - title: Cloud - group: 2 - type: group - footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' - fields: - - name: account.id - level: extended - type: keyword - description: |- - The cloud account or organization id used to identify different entities in a multi-tenant environment. - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. - ignore_above: 1024 - - name: account.name - level: extended - type: keyword - description: |- - The cloud account name or alias used to identify different entities in a multi-tenant environment. - Examples: AWS account name, Google Cloud ORG display name. - ignore_above: 1024 - - name: availability_zone - level: extended - type: keyword - description: Availability zone in which this host is running. - ignore_above: 1024 - - name: instance.id - level: extended - type: keyword - description: Instance ID of the host machine. - ignore_above: 1024 - - name: machine.type - level: extended - type: keyword - description: Machine type of the host machine. - ignore_above: 1024 - - name: provider - level: extended - type: keyword - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - ignore_above: 1024 - - name: region - level: extended - type: keyword - description: Region in which this host is running. - ignore_above: 1024 -- name: ecs.version - type: keyword - description: ECS version this event conforms to. - example: 1.0.0 - ignore_above: 1024 -- name: service.type - type: keyword - description: Service type +- external: ecs + name: cloud +- external: ecs + name: cloud.account.id +- external: ecs + name: cloud.account.name +- external: ecs + name: cloud.availability_zone +- external: ecs + name: cloud.instance.id +- external: ecs + name: cloud.machine.type +- external: ecs + name: cloud.provider +- external: ecs + name: cloud.region +- external: ecs + name: ecs.version +- external: ecs + name: error +- external: ecs + name: error.message +- external: ecs + name: service.type diff --git a/test/packages/aws/data_stream/natgateway/manifest.yml b/test/packages/aws/data_stream/natgateway/manifest.yml index fc6cf801c3..53dbac0141 100644 --- a/test/packages/aws/data_stream/natgateway/manifest.yml +++ b/test/packages/aws/data_stream/natgateway/manifest.yml @@ -1,5 +1,4 @@ title: AWS NAT gateway metrics -release: beta type: metrics streams: - input: aws/metrics diff --git a/test/packages/aws/data_stream/rds/agent/stream/stream.yml.hbs b/test/packages/aws/data_stream/rds/agent/stream/stream.yml.hbs index bf6deefcf8..0bafbe98c0 100644 --- a/test/packages/aws/data_stream/rds/agent/stream/stream.yml.hbs +++ b/test/packages/aws/data_stream/rds/agent/stream/stream.yml.hbs @@ -29,4 +29,7 @@ latency: {{latency}} {{/if}} {{#if tags_filter}} tags_filter: {{tags_filter}} +{{/if}} +{{#if proxy_url }} +proxy_url: {{proxy_url}} {{/if}} \ No newline at end of file diff --git a/test/packages/aws/data_stream/rds/fields/base-fields.yml b/test/packages/aws/data_stream/rds/fields/base-fields.yml index 7c798f4534..8166b56ec2 100644 --- a/test/packages/aws/data_stream/rds/fields/base-fields.yml +++ b/test/packages/aws/data_stream/rds/fields/base-fields.yml @@ -10,3 +10,11 @@ - name: '@timestamp' type: date description: Event timestamp. +- name: event.module + type: constant_keyword + description: Event module + value: aws +- name: event.dataset + type: constant_keyword + description: Event dataset + value: aws.rds diff --git a/test/packages/aws/data_stream/rds/fields/ecs.yml b/test/packages/aws/data_stream/rds/fields/ecs.yml index a02d7269c5..83e3f6f122 100644 --- a/test/packages/aws/data_stream/rds/fields/ecs.yml +++ b/test/packages/aws/data_stream/rds/fields/ecs.yml @@ -1,53 +1,24 @@ -- name: cloud - title: Cloud - group: 2 - type: group - footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' - fields: - - name: account.id - level: extended - type: keyword - description: |- - The cloud account or organization id used to identify different entities in a multi-tenant environment. - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. - ignore_above: 1024 - - name: account.name - level: extended - type: keyword - description: |- - The cloud account name or alias used to identify different entities in a multi-tenant environment. - Examples: AWS account name, Google Cloud ORG display name. - ignore_above: 1024 - - name: availability_zone - level: extended - type: keyword - description: Availability zone in which this host is running. - ignore_above: 1024 - - name: instance.id - level: extended - type: keyword - description: Instance ID of the host machine. - ignore_above: 1024 - - name: machine.type - level: extended - type: keyword - description: Machine type of the host machine. - ignore_above: 1024 - - name: provider - level: extended - type: keyword - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - ignore_above: 1024 - - name: region - level: extended - type: keyword - description: Region in which this host is running. - ignore_above: 1024 -- name: ecs.version - type: keyword - description: ECS version this event conforms to. - example: 1.0.0 - ignore_above: 1024 -- name: service.type - type: keyword - description: Service type +- external: ecs + name: cloud +- external: ecs + name: cloud.account.id +- external: ecs + name: cloud.account.name +- external: ecs + name: cloud.availability_zone +- external: ecs + name: cloud.instance.id +- external: ecs + name: cloud.machine.type +- external: ecs + name: cloud.provider +- external: ecs + name: cloud.region +- external: ecs + name: ecs.version +- external: ecs + name: error +- external: ecs + name: error.message +- external: ecs + name: service.type diff --git a/test/packages/aws/data_stream/rds/fields/fields.yml b/test/packages/aws/data_stream/rds/fields/fields.yml index 5f0deb4866..ba79cf1083 100644 --- a/test/packages/aws/data_stream/rds/fields/fields.yml +++ b/test/packages/aws/data_stream/rds/fields/fields.yml @@ -343,3 +343,9 @@ type: long description: | The remaining available space for the cluster volume, measured in bytes. + - name: cloudwatch + type: group + fields: + - name: namespace + type: keyword + description: The namespace specified when query cloudwatch api. diff --git a/test/packages/aws/data_stream/rds/manifest.yml b/test/packages/aws/data_stream/rds/manifest.yml index c920727e99..5632176c1c 100644 --- a/test/packages/aws/data_stream/rds/manifest.yml +++ b/test/packages/aws/data_stream/rds/manifest.yml @@ -1,5 +1,4 @@ title: AWS RDS metrics -release: beta type: metrics streams: - input: aws/metrics diff --git a/test/packages/aws/data_stream/s3_daily_storage/agent/stream/stream.yml.hbs b/test/packages/aws/data_stream/s3_daily_storage/agent/stream/stream.yml.hbs index cac1cae04a..eaee06ea7a 100644 --- a/test/packages/aws/data_stream/s3_daily_storage/agent/stream/stream.yml.hbs +++ b/test/packages/aws/data_stream/s3_daily_storage/agent/stream/stream.yml.hbs @@ -26,4 +26,7 @@ regions: {{/if}} {{#if latency}} latency: {{latency}} +{{/if}} +{{#if proxy_url }} +proxy_url: {{proxy_url}} {{/if}} \ No newline at end of file diff --git a/test/packages/aws/data_stream/s3_daily_storage/fields/base-fields.yml b/test/packages/aws/data_stream/s3_daily_storage/fields/base-fields.yml index 7c798f4534..57ae310ca3 100644 --- a/test/packages/aws/data_stream/s3_daily_storage/fields/base-fields.yml +++ b/test/packages/aws/data_stream/s3_daily_storage/fields/base-fields.yml @@ -10,3 +10,11 @@ - name: '@timestamp' type: date description: Event timestamp. +- name: event.module + type: constant_keyword + description: Event module + value: aws +- name: event.dataset + type: constant_keyword + description: Event dataset + value: aws.s3_daily_storage diff --git a/test/packages/aws/data_stream/s3_daily_storage/fields/ecs.yml b/test/packages/aws/data_stream/s3_daily_storage/fields/ecs.yml index a02d7269c5..83e3f6f122 100644 --- a/test/packages/aws/data_stream/s3_daily_storage/fields/ecs.yml +++ b/test/packages/aws/data_stream/s3_daily_storage/fields/ecs.yml @@ -1,53 +1,24 @@ -- name: cloud - title: Cloud - group: 2 - type: group - footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' - fields: - - name: account.id - level: extended - type: keyword - description: |- - The cloud account or organization id used to identify different entities in a multi-tenant environment. - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. - ignore_above: 1024 - - name: account.name - level: extended - type: keyword - description: |- - The cloud account name or alias used to identify different entities in a multi-tenant environment. - Examples: AWS account name, Google Cloud ORG display name. - ignore_above: 1024 - - name: availability_zone - level: extended - type: keyword - description: Availability zone in which this host is running. - ignore_above: 1024 - - name: instance.id - level: extended - type: keyword - description: Instance ID of the host machine. - ignore_above: 1024 - - name: machine.type - level: extended - type: keyword - description: Machine type of the host machine. - ignore_above: 1024 - - name: provider - level: extended - type: keyword - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - ignore_above: 1024 - - name: region - level: extended - type: keyword - description: Region in which this host is running. - ignore_above: 1024 -- name: ecs.version - type: keyword - description: ECS version this event conforms to. - example: 1.0.0 - ignore_above: 1024 -- name: service.type - type: keyword - description: Service type +- external: ecs + name: cloud +- external: ecs + name: cloud.account.id +- external: ecs + name: cloud.account.name +- external: ecs + name: cloud.availability_zone +- external: ecs + name: cloud.instance.id +- external: ecs + name: cloud.machine.type +- external: ecs + name: cloud.provider +- external: ecs + name: cloud.region +- external: ecs + name: ecs.version +- external: ecs + name: error +- external: ecs + name: error.message +- external: ecs + name: service.type diff --git a/test/packages/aws/data_stream/s3_daily_storage/manifest.yml b/test/packages/aws/data_stream/s3_daily_storage/manifest.yml index 89473f0ebb..f69e1889da 100644 --- a/test/packages/aws/data_stream/s3_daily_storage/manifest.yml +++ b/test/packages/aws/data_stream/s3_daily_storage/manifest.yml @@ -1,5 +1,4 @@ title: AWS S3 daily storage metrics -release: beta type: metrics streams: - input: aws/metrics diff --git a/test/packages/aws/data_stream/s3_request/agent/stream/stream.yml.hbs b/test/packages/aws/data_stream/s3_request/agent/stream/stream.yml.hbs index 6f53aab34d..80739aebc5 100644 --- a/test/packages/aws/data_stream/s3_request/agent/stream/stream.yml.hbs +++ b/test/packages/aws/data_stream/s3_request/agent/stream/stream.yml.hbs @@ -26,4 +26,7 @@ regions: {{/if}} {{#if latency}} latency: {{latency}} +{{/if}} +{{#if proxy_url }} +proxy_url: {{proxy_url}} {{/if}} \ No newline at end of file diff --git a/test/packages/aws/data_stream/s3_request/fields/base-fields.yml b/test/packages/aws/data_stream/s3_request/fields/base-fields.yml index 7c798f4534..e0956c2aab 100644 --- a/test/packages/aws/data_stream/s3_request/fields/base-fields.yml +++ b/test/packages/aws/data_stream/s3_request/fields/base-fields.yml @@ -10,3 +10,11 @@ - name: '@timestamp' type: date description: Event timestamp. +- name: event.module + type: constant_keyword + description: Event module + value: aws +- name: event.dataset + type: constant_keyword + description: Event dataset + value: aws.s3_request diff --git a/test/packages/aws/data_stream/s3_request/fields/ecs.yml b/test/packages/aws/data_stream/s3_request/fields/ecs.yml index a02d7269c5..83e3f6f122 100644 --- a/test/packages/aws/data_stream/s3_request/fields/ecs.yml +++ b/test/packages/aws/data_stream/s3_request/fields/ecs.yml @@ -1,53 +1,24 @@ -- name: cloud - title: Cloud - group: 2 - type: group - footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' - fields: - - name: account.id - level: extended - type: keyword - description: |- - The cloud account or organization id used to identify different entities in a multi-tenant environment. - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. - ignore_above: 1024 - - name: account.name - level: extended - type: keyword - description: |- - The cloud account name or alias used to identify different entities in a multi-tenant environment. - Examples: AWS account name, Google Cloud ORG display name. - ignore_above: 1024 - - name: availability_zone - level: extended - type: keyword - description: Availability zone in which this host is running. - ignore_above: 1024 - - name: instance.id - level: extended - type: keyword - description: Instance ID of the host machine. - ignore_above: 1024 - - name: machine.type - level: extended - type: keyword - description: Machine type of the host machine. - ignore_above: 1024 - - name: provider - level: extended - type: keyword - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - ignore_above: 1024 - - name: region - level: extended - type: keyword - description: Region in which this host is running. - ignore_above: 1024 -- name: ecs.version - type: keyword - description: ECS version this event conforms to. - example: 1.0.0 - ignore_above: 1024 -- name: service.type - type: keyword - description: Service type +- external: ecs + name: cloud +- external: ecs + name: cloud.account.id +- external: ecs + name: cloud.account.name +- external: ecs + name: cloud.availability_zone +- external: ecs + name: cloud.instance.id +- external: ecs + name: cloud.machine.type +- external: ecs + name: cloud.provider +- external: ecs + name: cloud.region +- external: ecs + name: ecs.version +- external: ecs + name: error +- external: ecs + name: error.message +- external: ecs + name: service.type diff --git a/test/packages/aws/data_stream/s3_request/manifest.yml b/test/packages/aws/data_stream/s3_request/manifest.yml index 472461c764..d02b858648 100644 --- a/test/packages/aws/data_stream/s3_request/manifest.yml +++ b/test/packages/aws/data_stream/s3_request/manifest.yml @@ -1,5 +1,4 @@ title: AWS S3 request metrics -release: beta type: metrics streams: - input: aws/metrics diff --git a/test/packages/aws/data_stream/s3_storage_lens/agent/stream/stream.yml.hbs b/test/packages/aws/data_stream/s3_storage_lens/agent/stream/stream.yml.hbs new file mode 100644 index 0000000000..87fbbb280f --- /dev/null +++ b/test/packages/aws/data_stream/s3_storage_lens/agent/stream/stream.yml.hbs @@ -0,0 +1,101 @@ +metricsets: ["cloudwatch"] +period: {{period}} +{{#if access_key_id}} +access_key_id: {{access_key_id}} +{{/if}} +{{#if secret_access_key}} +secret_access_key: {{secret_access_key}} +{{/if}} +{{#if session_token}} +session_token: {{session_token}} +{{/if}} +{{#if credential_profile_name}} +credential_profile_name: {{credential_profile_name}} +{{/if}} +{{#if shared_credential_file}} +shared_credential_file: {{shared_credential_file}} +{{/if}} +{{#if role_arn}} +role_arn: {{role_arn}} +{{/if}} +{{#if regions}} +regions: +{{#each regions as |region i|}} +- {{region}} +{{/each}} +{{/if}} +{{#if latency}} +latency: {{latency}} +{{/if}} +{{#if proxy_url }} +proxy_url: {{proxy_url}} +{{/if}} +metrics: +- namespace: "AWS/S3/Storage-Lens" + statistic: ["Average"] +processors: + - rename: + ignore_missing: true + fields: + - from: "aws.storage-lens.metrics.4xxErrors.avg" + to: "aws.s3_storage_lens.metrics.4xxErrors.avg" + - from: "aws.storage-lens.metrics.5xxErrors.avg" + to: "aws.s3_storage_lens.metrics.5xxErrors.avg" + - from: "aws.storage-lens.metrics.AllRequests.avg" + to: "aws.s3_storage_lens.metrics.AllRequests.avg" + - from: "aws.storage-lens.metrics.BytesDownloaded.avg" + to: "aws.s3_storage_lens.metrics.BytesDownloaded.avg" + - from: "aws.storage-lens.metrics.BytesUploaded.avg" + to: "aws.s3_storage_lens.metrics.BytesUploaded.avg" + - from: "aws.storage-lens.metrics.CurrentVersionObjectCount.avg" + to: "aws.s3_storage_lens.metrics.CurrentVersionObjectCount.avg" + - from: "aws.storage-lens.metrics.CurrentVersionStorageBytes.avg" + to: "aws.s3_storage_lens.metrics.CurrentVersionStorageBytes.avg" + - from: "aws.storage-lens.metrics.DeleteMarkerObjectCount.avg" + to: "aws.s3_storage_lens.metrics.DeleteMarkerObjectCount.avg" + - from: "aws.storage-lens.metrics.DeleteRequests.avg" + to: "aws.s3_storage_lens.metrics.DeleteRequests.avg" + - from: "aws.storage-lens.metrics.EncryptedObjectCount.avg" + to: "aws.s3_storage_lens.metrics.EncryptedObjectCount.avg" + - from: "aws.storage-lens.metrics.EncryptedStorageBytes.avg" + to: "aws.s3_storage_lens.metrics.EncryptedStorageBytes.avg" + - from: "aws.storage-lens.metrics.GetRequests.avg" + to: "aws.s3_storage_lens.metrics.GetRequests.avg" + - from: "aws.storage-lens.metrics.HeadRequests.avg" + to: "aws.s3_storage_lens.metrics.HeadRequests.avg" + - from: "aws.storage-lens.metrics.IncompleteMultipartUploadObjectCount.avg" + to: "aws.s3_storage_lens.metrics.IncompleteMultipartUploadObjectCount.avg" + - from: "aws.storage-lens.metrics.IncompleteMultipartUploadStorageBytes.avg" + to: "aws.s3_storage_lens.metrics.IncompleteMultipartUploadStorageBytes.avg" + - from: "aws.storage-lens.metrics.ListRequests.avg" + to: "aws.s3_storage_lens.metrics.ListRequests.avg" + - from: "aws.storage-lens.metrics.NonCurrentVersionObjectCount.avg" + to: "aws.s3_storage_lens.metrics.NonCurrentVersionObjectCount.avg" + - from: "aws.storage-lens.metrics.NonCurrentVersionStorageBytes.avg" + to: "aws.s3_storage_lens.metrics.NonCurrentVersionStorageBytes.avg" + - from: "aws.storage-lens.metrics.ObjectCount.avg" + to: "aws.s3_storage_lens.metrics.ObjectCount.avg" + - from: "aws.storage-lens.metrics.ObjectLockEnabledObjectCount.avg" + to: "aws.s3_storage_lens.metrics.ObjectLockEnabledObjectCount.avg" + - from: "aws.storage-lens.metrics.ObjectLockEnabledStorageBytes.avg" + to: "aws.s3_storage_lens.metrics.ObjectLockEnabledStorageBytes.avg" + - from: "aws.storage-lens.metrics.PostRequests.avg" + to: "aws.s3_storage_lens.metrics.PostRequests.avg" + - from: "aws.storage-lens.metrics.PutRequests.avg" + to: "aws.s3_storage_lens.metrics.PutRequests.avg" + - from: "aws.storage-lens.metrics.ReplicatedObjectCount.avg" + to: "aws.s3_storage_lens.metrics.ReplicatedObjectCount.avg" + - from: "aws.storage-lens.metrics.ReplicatedStorageBytes.avg" + to: "aws.s3_storage_lens.metrics.ReplicatedStorageBytes.avg" + - from: "aws.storage-lens.metrics.SelectRequests.avg" + to: "aws.s3_storage_lens.metrics.SelectRequests.avg" + - from: "aws.storage-lens.metrics.SelectReturnedBytes.avg" + to: "aws.s3_storage_lens.metrics.SelectReturnedBytes.avg" + - from: "aws.storage-lens.metrics.SelectScannedBytes.avg" + to: "aws.s3_storage_lens.metrics.SelectScannedBytes.avg" + - from: "aws.storage-lens.metrics.StorageBytes.avg" + to: "aws.s3_storage_lens.metrics.StorageBytes.avg" + - drop_fields: + ignore_missing: true + fields: + - "aws.storage-lens" \ No newline at end of file diff --git a/test/packages/aws/data_stream/s3_storage_lens/fields/agent.yml b/test/packages/aws/data_stream/s3_storage_lens/fields/agent.yml new file mode 100644 index 0000000000..da4e652c53 --- /dev/null +++ b/test/packages/aws/data_stream/s3_storage_lens/fields/agent.yml @@ -0,0 +1,198 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + diff --git a/test/packages/aws/data_stream/s3_storage_lens/fields/base-fields.yml b/test/packages/aws/data_stream/s3_storage_lens/fields/base-fields.yml new file mode 100644 index 0000000000..ed9d40b9d6 --- /dev/null +++ b/test/packages/aws/data_stream/s3_storage_lens/fields/base-fields.yml @@ -0,0 +1,20 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. +- name: event.module + type: constant_keyword + description: Event module + value: aws +- name: event.dataset + type: constant_keyword + description: Event dataset + value: aws.s3_storage_lens diff --git a/test/packages/aws/data_stream/s3_storage_lens/fields/ecs.yml b/test/packages/aws/data_stream/s3_storage_lens/fields/ecs.yml new file mode 100644 index 0000000000..83e3f6f122 --- /dev/null +++ b/test/packages/aws/data_stream/s3_storage_lens/fields/ecs.yml @@ -0,0 +1,24 @@ +- external: ecs + name: cloud +- external: ecs + name: cloud.account.id +- external: ecs + name: cloud.account.name +- external: ecs + name: cloud.availability_zone +- external: ecs + name: cloud.instance.id +- external: ecs + name: cloud.machine.type +- external: ecs + name: cloud.provider +- external: ecs + name: cloud.region +- external: ecs + name: ecs.version +- external: ecs + name: error +- external: ecs + name: error.message +- external: ecs + name: service.type diff --git a/test/packages/aws/data_stream/s3_storage_lens/fields/fields.yml b/test/packages/aws/data_stream/s3_storage_lens/fields/fields.yml new file mode 100644 index 0000000000..d1230dcc54 --- /dev/null +++ b/test/packages/aws/data_stream/s3_storage_lens/fields/fields.yml @@ -0,0 +1,100 @@ +- name: aws + type: group + release: experimental + fields: + - name: s3_storage_lens + type: group + fields: + - name: metrics + type: group + fields: + - name: 4xxErrors.avg + type: long + description: The total 4xx errors in scope. + - name: 5xxErrors.avg + type: long + description: The total 5xx errors in scope. + - name: AllRequests.avg + type: long + description: The total number of requests made. + - name: BytesDownloaded.avg + type: long + description: The number of bytes in scope that were downloaded. + - name: BytesUploaded.avg + type: long + description: The number of bytes uploaded. + - name: CurrentVersionObjectCount.avg + type: long + description: The number of objects that are a current version. + - name: CurrentVersionStorageBytes.avg + type: long + description: The number of bytes that are a current version. + - name: DeleteMarkerObjectCount.avg + type: long + description: The total number of objects with a delete marker. + - name: DeleteRequests.avg + type: long + description: The total number of delete requests made. + - name: EncryptedObjectCount.avg + type: long + description: The total object counts that are encrypted using Amazon S3 server-side encryption. + - name: EncryptedStorageBytes.avg + type: long + description: The total number of encrypted bytes using Amazon S3 server-side encryption. + - name: GetRequests.avg + type: long + description: The total number of GET requests made. + - name: HeadRequests.avg + type: long + description: The total number of head requests made. + - name: IncompleteMultipartUploadObjectCount.avg + type: long + description: The number of objects in scope that are incomplete multipart uploads. + - name: IncompleteMultipartUploadStorageBytes.avg + type: long + description: The total bytes in scope with incomplete multipart uploads. + - name: ListRequests.avg + type: long + description: The total number of list requests made. + - name: NonCurrentVersionObjectCount.avg + type: long + description: The count of the noncurrent version objects. + - name: NonCurrentVersionStorageBytes.avg + type: long + description: The number of noncurrent versioned bytes. + - name: ObjectCount.avg + type: long + description: The total object count. + - name: ObjectLockEnabledObjectCount.avg + type: long + description: The total number of objects in scope that have Object Lock enabled. + - name: ObjectLockEnabledStorageBytes.avg + type: long + description: The total number of bytes in scope that have Object Lock enabled. + - name: PostRequests.avg + type: long + description: The total number of post requests made. + - name: PutRequests.avg + type: long + description: The total number of PUT requests made. + - name: ReplicatedObjectCount.avg + type: long + description: The count of replicated objects. + - name: ReplicatedStorageBytes.avg + type: long + description: The total number of bytes in scope that are replicated. + - name: SelectRequests.avg + type: long + description: The total number of select requests. + - name: SelectReturnedBytes.avg + type: long + description: The number of select bytes returned. + - name: SelectScannedBytes.avg + type: long + description: The number of select bytes scanned. + - name: StorageBytes.avg + type: long + description: The total storage in bytes +- name: aws.cloudwatch.namespace + type: keyword + description: The namespace specified when query cloudwatch api. diff --git a/test/packages/aws/data_stream/s3_storage_lens/fields/package-fields.yml b/test/packages/aws/data_stream/s3_storage_lens/fields/package-fields.yml new file mode 100644 index 0000000000..a8a7ee8dcc --- /dev/null +++ b/test/packages/aws/data_stream/s3_storage_lens/fields/package-fields.yml @@ -0,0 +1,19 @@ +- name: aws + type: group + fields: + - name: tags.* + type: object + description: | + Tag key value pairs from aws resources. + - name: s3.bucket.name + type: keyword + description: | + Name of a S3 bucket. + - name: dimensions.* + type: object + description: | + Metric dimensions. + - name: '*.metrics.*.*' + type: object + description: | + Metrics that returned from Cloudwatch API query. diff --git a/test/packages/aws/data_stream/s3_storage_lens/manifest.yml b/test/packages/aws/data_stream/s3_storage_lens/manifest.yml new file mode 100644 index 0000000000..b325ed0cb8 --- /dev/null +++ b/test/packages/aws/data_stream/s3_storage_lens/manifest.yml @@ -0,0 +1,26 @@ +title: AWS S3 Storage Lens metrics +type: metrics +streams: + - input: aws/metrics + vars: + - name: period + type: text + title: Period + multi: false + required: true + show_user: true + default: 24h + - name: regions + type: text + title: Regions + multi: true + required: false + show_user: true + - name: latency + type: text + title: Latency + multi: false + required: false + show_user: false + title: AWS S3 Storage Lens metrics + description: Collect AWS S3 Storage Lens metrics diff --git a/test/packages/aws/data_stream/s3_storage_lens/sample_event.json b/test/packages/aws/data_stream/s3_storage_lens/sample_event.json new file mode 100644 index 0000000000..dbc4ccc9cf --- /dev/null +++ b/test/packages/aws/data_stream/s3_storage_lens/sample_event.json @@ -0,0 +1,138 @@ +{ + "@timestamp": "2021-11-07T20:38:00.000Z", + "ecs": { + "version": "1.11.0" + }, + "data_stream": { + "namespace": "default", + "type": "metrics", + "dataset": "aws.s3_storage_lens" + }, + "service": { + "type": "aws" + }, + "cloud": { + "provider": "aws", + "region": "us-east-1", + "account": { + "name": "elastic-beats", + "id": "428152502467" + } + }, + "metricset": { + "period": 86400000, + "name": "cloudwatch" + }, + "event": { + "duration": 22973251900, + "agent_id_status": "verified", + "ingested": "2021-11-08T20:38:37Z", + "module": "aws", + "dataset": "aws.s3_storage_lens" + }, + "aws": { + "s3_storage_lens": { + "metrics": { + "NonCurrentVersionStorageBytes": { + "avg": 0 + }, + "DeleteMarkerObjectCount": { + "avg": 0 + }, + "GetRequests": { + "avg": 0 + }, + "SelectReturnedBytes": { + "avg": 0 + }, + "ObjectCount": { + "avg": 164195 + }, + "HeadRequests": { + "avg": 0 + }, + "ListRequests": { + "avg": 0 + }, + "DeleteRequests": { + "avg": 0 + }, + "SelectRequests": { + "avg": 0 + }, + "5xxErrors": { + "avg": 0 + }, + "BytesDownloaded": { + "avg": 0 + }, + "BytesUploaded": { + "avg": 82537 + }, + "CurrentVersionStorageBytes": { + "avg": 154238334 + }, + "StorageBytes": { + "avg": 154238334 + }, + "ObjectLockEnabledStorageBytes": { + "avg": 0 + }, + "4xxErrors": { + "avg": 0 + }, + "PutRequests": { + "avg": 145 + }, + "ObjectLockEnabledObjectCount": { + "avg": 0 + }, + "EncryptedObjectCount": { + "avg": 164191 + }, + "CurrentVersionObjectCount": { + "avg": 164195 + }, + "IncompleteMultipartUploadObjectCount": { + "avg": 0 + }, + "ReplicatedObjectCount": { + "avg": 0 + }, + "AllRequests": { + "avg": 145 + }, + "PostRequests": { + "avg": 0 + }, + "IncompleteMultipartUploadStorageBytes": { + "avg": 0 + }, + "NonCurrentVersionObjectCount": { + "avg": 0 + }, + "ReplicatedStorageBytes": { + "avg": 0 + }, + "EncryptedStorageBytes": { + "avg": 154237917 + }, + "SelectScannedBytes": { + "avg": 0 + } + } + }, + "cloudwatch": { + "namespace": "AWS/S3/Storage-Lens" + }, + "dimensions": { + "metrics_version": "1.0", + "storage_class": "STANDARD", + "aws_region": "eu-central-1", + "bucket_name": "filebeat-aws-elb-test", + "aws_account_number": "428152502467", + "configuration_id": "default-account-dashboard", + "record_type": "BUCKET" + } + } +} \ No newline at end of file diff --git a/test/packages/aws/data_stream/s3access/_dev/test/pipeline/test-common-config.yml b/test/packages/aws/data_stream/s3access/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 0000000000..5622947e4b --- /dev/null +++ b/test/packages/aws/data_stream/s3access/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,5 @@ +dynamic_fields: + event.ingested: ".*" +fields: + tags: + - preserve_original_event diff --git a/test/packages/aws/data_stream/s3access/_dev/test/pipeline/test-s3-server-access.log b/test/packages/aws/data_stream/s3access/_dev/test/pipeline/test-s3-server-access.log index f96091a767..bcc9f6af0d 100644 --- a/test/packages/aws/data_stream/s3access/_dev/test/pipeline/test-s3-server-access.log +++ b/test/packages/aws/data_stream/s3access/_dev/test/pipeline/test-s3-server-access.log @@ -1,6 +1,7 @@ -36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:41 +0000] 72.21.217.31 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 44EE8651683CB4DA REST.GET.LOCATION - "GET /test-s3-ks/?location&aws-account=627959692251 HTTP/1.1" 200 - 142 - 17 - "-" "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation" - BsCfJedfuSnds2QFoxi+E/O7M6OEWzJnw4dUaes/2hyA363sONRJKzB7EOY+Bt9DTHYUn+HoHxI= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2 -36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:42 +0000] 72.21.217.31 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 E26222010BCC32B6 REST.GET.LOCATION - "GET /test-s3-ks/?location&aws-account=627959692251 HTTP/1.1" 200 - 142 - 3 - "-" "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation" - gNl/Q1IzY6nGTBygqI3rnMz/ZFOFwOTDpSMrNca+IcEmMAd6sCIs1ZRLYDekD8LB9lrj9UdQLWE= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2 -36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:43 +0000] 72.21.217.31 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 4DD6D17D1C5C401C REST.GET.BUCKET - "GET /test-s3-ks/?max-keys=0&encoding-type=url&aws-account=627959692251 HTTP/1.1" 200 - 265 - 2 1 "-" "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation" - KzvchfojYQnuFC4PABYVJVxIlv/f6r17LRaTSvw7x+bxj4PkkPKT1kX9x8wbqtq40iD4PC881iE= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2 -36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:43 +0000] 72.21.217.31 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 706992E2F3CC3C3D REST.GET.LOCATION - "GET /test-s3-ks/?location&aws-account=627959692251 HTTP/1.1" 200 - 142 - 4 - "-" "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation" - cIN12KTrJwx+uTBZD+opZUPE4iGypi8oG/oXGPzFk9CMuHQGuEpmAeNELdtYKDxf2TDor25Nikg= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2 -36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 jsoriano-s3-test [10/Sep/2019:15:11:07 +0000] 77.227.156.41 arn:aws:iam::123456:user/test@elastic.co 8CD7A4A71E2E5C9E BATCH.DELETE.OBJECT jolokia-war-1.5.0.war - 204 - - 344017 - - - - - IeDW5I3wefFxU8iHOcAzi5qr+O+1bdRlcQ0AO2WGjFh7JwYM6qCoKq+1TrUshrXMlBxPFtg97Vk= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.eu-central-1.amazonaws.com TLSv1.2 -36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [19/Sep/2019:17:06:39 +0000] 174.29.206.152 arn:aws:iam::123456:user/test@elastic.co 6CE38F1312D32BDD BATCH.DELETE.OBJECT Screen+Shot+2019-09-09+at+9.08.44+AM.png - 204 - - 57138 - - - - - LwRa4w6DbuU48GKQiH3jDbjfTyLCbwasFBsdttugRQ+9lH4jK8lT91+HhGZKMYI3sPyKuQ9LvU0= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3-ap-southeast-1.amazonaws.com TLSv1.2 +36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:41 +0000] 89.160.20.156 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 44EE8651683CB4DA REST.GET.LOCATION - "GET /test-s3-ks/?location&aws-account=627959692251 HTTP/1.1" 200 - 142 - 17 - "-" "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation" - BsCfJedfuSnds2QFoxi+E/O7M6OEWzJnw4dUaes/2hyA363sONRJKzB7EOY+Bt9DTHYUn+HoHxI= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2 +36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:42 +0000] 89.160.20.156 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 E26222010BCC32B6 REST.GET.LOCATION - "GET /test-s3-ks/?location&aws-account=627959692251 HTTP/1.1" 200 - 142 - 3 - "-" "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation" - gNl/Q1IzY6nGTBygqI3rnMz/ZFOFwOTDpSMrNca+IcEmMAd6sCIs1ZRLYDekD8LB9lrj9UdQLWE= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2 +36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:43 +0000] 89.160.20.156 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 4DD6D17D1C5C401C REST.GET.BUCKET - "GET /test-s3-ks/?max-keys=0&encoding-type=url&aws-account=627959692251 HTTP/1.1" 200 - 265 - 2 1 "-" "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation" - KzvchfojYQnuFC4PABYVJVxIlv/f6r17LRaTSvw7x+bxj4PkkPKT1kX9x8wbqtq40iD4PC881iE= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2 +36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:43 +0000] 89.160.20.156 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 706992E2F3CC3C3D REST.GET.LOCATION - "GET /test-s3-ks/?location&aws-account=627959692251 HTTP/1.1" 200 - 142 - 4 - "-" "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation" - cIN12KTrJwx+uTBZD+opZUPE4iGypi8oG/oXGPzFk9CMuHQGuEpmAeNELdtYKDxf2TDor25Nikg= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2 +36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 jsoriano-s3-test [10/Sep/2019:15:11:07 +0000] 89.160.20.156 arn:aws:iam::123456:user/test@elastic.co 8CD7A4A71E2E5C9E BATCH.DELETE.OBJECT jolokia-war-1.5.0.war - 204 - - 344017 - - - - - IeDW5I3wefFxU8iHOcAzi5qr+O+1bdRlcQ0AO2WGjFh7JwYM6qCoKq+1TrUshrXMlBxPFtg97Vk= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.eu-central-1.amazonaws.com TLSv1.2 +36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [19/Sep/2019:17:06:39 +0000] 89.160.20.156 arn:aws:iam::123456:user/test@elastic.co 6CE38F1312D32BDD BATCH.DELETE.OBJECT Screen+Shot+2019-09-09+at+9.08.44+AM.png - 204 - - 57138 - - - - - LwRa4w6DbuU48GKQiH3jDbjfTyLCbwasFBsdttugRQ+9lH4jK8lT91+HhGZKMYI3sPyKuQ9LvU0= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2 +67797214d75628047d9c76b18a78cded1a4b069b71f2a9d5a53649c38da8770b flow-log-test [14/Jul/2021:18:57:31 +0000] - svc:delivery.logs.amazonaws.com MVGXZXEVN3IG9S24 REST.PUT.OBJECT AWSLogs/000000000000/vpcflowlogs/us-gov-east-1/2021/07/13/000000000000_vpcflowlogs_us-gov-east-1_fl-_20210713T1855Z_f12aa632.log.gz "PUT /AWSLogs/000000000000/vpcflowlogs/us-gov-east-1/2021/07/13/000000000000_vpcflowlogs_us-gov-east-1_fl-0e7c13bf00cf15bfe_20210713T1855Z_f12aa632.log.gz HTTP/1.1" 200 - - 773 103 13 "-" "-" - 02SxwfXpO5UysN0GsKGa3uGDQ6E/W7+Hwo/luRH8p1VEexULoe66RCM+nja0dEq2JqLrtgjocvVRRkVt4= SigV4 ECDHE-RSA-AES128-GCM-SHA256 AuthHeader flow-log-test.s3.us-gov-west-1.amazonaws.com TLSv1.2 - diff --git a/test/packages/aws/data_stream/s3access/_dev/test/pipeline/test-s3-server-access.log-config.yml b/test/packages/aws/data_stream/s3access/_dev/test/pipeline/test-s3-server-access.log-config.yml deleted file mode 100644 index c39dc38617..0000000000 --- a/test/packages/aws/data_stream/s3access/_dev/test/pipeline/test-s3-server-access.log-config.yml +++ /dev/null @@ -1,2 +0,0 @@ -dynamic_fields: - event.ingested: ".*" diff --git a/test/packages/aws/data_stream/s3access/_dev/test/pipeline/test-s3-server-access.log-expected.json b/test/packages/aws/data_stream/s3access/_dev/test/pipeline/test-s3-server-access.log-expected.json index 214d8e0d35..abd3a9e475 100644 --- a/test/packages/aws/data_stream/s3access/_dev/test/pipeline/test-s3-server-access.log-expected.json +++ b/test/packages/aws/data_stream/s3access/_dev/test/pipeline/test-s3-server-access.log-expected.json @@ -6,28 +6,33 @@ "original": "/test-s3-ks/?location\u0026aws-account=627959692251", "query": "location\u0026aws-account=627959692251" }, + "tags": [ + "preserve_original_event" + ], "geo": { - "continent_name": "North America", - "region_iso_code": "US-VA", - "city_name": "Ashburn", - "country_iso_code": "US", - "country_name": "United States", - "region_name": "Virginia", + "continent_name": "Europe", + "region_iso_code": "SE-AB", + "city_name": "Tumba", + "country_iso_code": "SE", + "country_name": "Sweden", + "region_name": "Stockholm", "location": { - "lon": -77.4728, - "lat": 39.0481 + "lon": 17.8167, + "lat": 59.2 } }, "cloud": { "provider": "aws" }, - "@timestamp": "2019-08-01T00:24:41.000Z", + "ecs": { + "version": "1.12.0" + }, "related": { "user": [ "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2" ], "ip": [ - "72.21.217.31" + "89.160.20.156" ] }, "http": { @@ -46,8 +51,8 @@ "user": { "id": "arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9" }, - "address": "72.21.217.31", - "ip": "72.21.217.31" + "address": "89.160.20.156", + "ip": "89.160.20.156" }, "tls": { "cipher": "ECDHE-RSA-AES128-SHA", @@ -56,8 +61,8 @@ }, "event": { "duration": 17000000, - "ingested": "2021-04-23T12:15:55.855305051Z", - "original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:41 +0000] 72.21.217.31 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 44EE8651683CB4DA REST.GET.LOCATION - \"GET /test-s3-ks/?location\u0026aws-account=627959692251 HTTP/1.1\" 200 - 142 - 17 - \"-\" \"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation\" - BsCfJedfuSnds2QFoxi+E/O7M6OEWzJnw4dUaes/2hyA363sONRJKzB7EOY+Bt9DTHYUn+HoHxI= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2", + "ingested": "2021-12-09T16:11:59.134194800Z", + "original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:41 +0000] 89.160.20.156 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 44EE8651683CB4DA REST.GET.LOCATION - \"GET /test-s3-ks/?location\u0026aws-account=627959692251 HTTP/1.1\" 200 - 142 - 17 - \"-\" \"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation\" - BsCfJedfuSnds2QFoxi+E/O7M6OEWzJnw4dUaes/2hyA363sONRJKzB7EOY+Bt9DTHYUn+HoHxI= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2", "kind": "event", "action": "REST.GET.LOCATION", "id": "44EE8651683CB4DA", @@ -78,7 +83,7 @@ "host_id": "BsCfJedfuSnds2QFoxi+E/O7M6OEWzJnw4dUaes/2hyA363sONRJKzB7EOY+Bt9DTHYUn+HoHxI=", "host_header": "s3.ap-southeast-1.amazonaws.com", "bucket": "test-s3-ks", - "remote_ip": "72.21.217.31", + "remote_ip": "89.160.20.156", "cipher_suite": "ECDHE-RSA-AES128-SHA", "http_status": 200, "total_time": 17, @@ -108,28 +113,33 @@ "original": "/test-s3-ks/?location\u0026aws-account=627959692251", "query": "location\u0026aws-account=627959692251" }, + "tags": [ + "preserve_original_event" + ], "geo": { - "continent_name": "North America", - "region_iso_code": "US-VA", - "city_name": "Ashburn", - "country_iso_code": "US", - "country_name": "United States", - "region_name": "Virginia", + "continent_name": "Europe", + "region_iso_code": "SE-AB", + "city_name": "Tumba", + "country_iso_code": "SE", + "country_name": "Sweden", + "region_name": "Stockholm", "location": { - "lon": -77.4728, - "lat": 39.0481 + "lon": 17.8167, + "lat": 59.2 } }, "cloud": { "provider": "aws" }, - "@timestamp": "2019-08-01T00:24:42.000Z", + "ecs": { + "version": "1.12.0" + }, "related": { "user": [ "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2" ], "ip": [ - "72.21.217.31" + "89.160.20.156" ] }, "http": { @@ -148,8 +158,8 @@ "user": { "id": "arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9" }, - "address": "72.21.217.31", - "ip": "72.21.217.31" + "address": "89.160.20.156", + "ip": "89.160.20.156" }, "tls": { "cipher": "ECDHE-RSA-AES128-SHA", @@ -158,8 +168,8 @@ }, "event": { "duration": 3000000, - "ingested": "2021-04-23T12:15:55.855312724Z", - "original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:42 +0000] 72.21.217.31 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 E26222010BCC32B6 REST.GET.LOCATION - \"GET /test-s3-ks/?location\u0026aws-account=627959692251 HTTP/1.1\" 200 - 142 - 3 - \"-\" \"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation\" - gNl/Q1IzY6nGTBygqI3rnMz/ZFOFwOTDpSMrNca+IcEmMAd6sCIs1ZRLYDekD8LB9lrj9UdQLWE= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2", + "ingested": "2021-12-09T16:11:59.134198700Z", + "original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:42 +0000] 89.160.20.156 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 E26222010BCC32B6 REST.GET.LOCATION - \"GET /test-s3-ks/?location\u0026aws-account=627959692251 HTTP/1.1\" 200 - 142 - 3 - \"-\" \"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation\" - gNl/Q1IzY6nGTBygqI3rnMz/ZFOFwOTDpSMrNca+IcEmMAd6sCIs1ZRLYDekD8LB9lrj9UdQLWE= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2", "kind": "event", "action": "REST.GET.LOCATION", "id": "E26222010BCC32B6", @@ -180,7 +190,7 @@ "host_id": "gNl/Q1IzY6nGTBygqI3rnMz/ZFOFwOTDpSMrNca+IcEmMAd6sCIs1ZRLYDekD8LB9lrj9UdQLWE=", "host_header": "s3.ap-southeast-1.amazonaws.com", "bucket": "test-s3-ks", - "remote_ip": "72.21.217.31", + "remote_ip": "89.160.20.156", "cipher_suite": "ECDHE-RSA-AES128-SHA", "http_status": 200, "total_time": 3, @@ -210,28 +220,33 @@ "original": "/test-s3-ks/?max-keys=0\u0026encoding-type=url\u0026aws-account=627959692251", "query": "max-keys=0\u0026encoding-type=url\u0026aws-account=627959692251" }, + "tags": [ + "preserve_original_event" + ], "geo": { - "continent_name": "North America", - "region_iso_code": "US-VA", - "city_name": "Ashburn", - "country_iso_code": "US", - "country_name": "United States", - "region_name": "Virginia", + "continent_name": "Europe", + "region_iso_code": "SE-AB", + "city_name": "Tumba", + "country_iso_code": "SE", + "country_name": "Sweden", + "region_name": "Stockholm", "location": { - "lon": -77.4728, - "lat": 39.0481 + "lon": 17.8167, + "lat": 59.2 } }, "cloud": { "provider": "aws" }, - "@timestamp": "2019-08-01T00:24:43.000Z", + "ecs": { + "version": "1.12.0" + }, "related": { "user": [ "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2" ], "ip": [ - "72.21.217.31" + "89.160.20.156" ] }, "http": { @@ -250,8 +265,8 @@ "user": { "id": "arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9" }, - "address": "72.21.217.31", - "ip": "72.21.217.31" + "address": "89.160.20.156", + "ip": "89.160.20.156" }, "tls": { "cipher": "ECDHE-RSA-AES128-SHA", @@ -260,8 +275,8 @@ }, "event": { "duration": 2000000, - "ingested": "2021-04-23T12:15:55.855315024Z", - "original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:43 +0000] 72.21.217.31 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 4DD6D17D1C5C401C REST.GET.BUCKET - \"GET /test-s3-ks/?max-keys=0\u0026encoding-type=url\u0026aws-account=627959692251 HTTP/1.1\" 200 - 265 - 2 1 \"-\" \"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation\" - KzvchfojYQnuFC4PABYVJVxIlv/f6r17LRaTSvw7x+bxj4PkkPKT1kX9x8wbqtq40iD4PC881iE= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2", + "ingested": "2021-12-09T16:11:59.134204100Z", + "original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:43 +0000] 89.160.20.156 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 4DD6D17D1C5C401C REST.GET.BUCKET - \"GET /test-s3-ks/?max-keys=0\u0026encoding-type=url\u0026aws-account=627959692251 HTTP/1.1\" 200 - 265 - 2 1 \"-\" \"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation\" - KzvchfojYQnuFC4PABYVJVxIlv/f6r17LRaTSvw7x+bxj4PkkPKT1kX9x8wbqtq40iD4PC881iE= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2", "kind": "event", "action": "REST.GET.BUCKET", "id": "4DD6D17D1C5C401C", @@ -283,7 +298,7 @@ "host_id": "KzvchfojYQnuFC4PABYVJVxIlv/f6r17LRaTSvw7x+bxj4PkkPKT1kX9x8wbqtq40iD4PC881iE=", "host_header": "s3.ap-southeast-1.amazonaws.com", "bucket": "test-s3-ks", - "remote_ip": "72.21.217.31", + "remote_ip": "89.160.20.156", "cipher_suite": "ECDHE-RSA-AES128-SHA", "http_status": 200, "total_time": 2, @@ -313,28 +328,33 @@ "original": "/test-s3-ks/?location\u0026aws-account=627959692251", "query": "location\u0026aws-account=627959692251" }, + "tags": [ + "preserve_original_event" + ], "geo": { - "continent_name": "North America", - "region_iso_code": "US-VA", - "city_name": "Ashburn", - "country_iso_code": "US", - "country_name": "United States", - "region_name": "Virginia", + "continent_name": "Europe", + "region_iso_code": "SE-AB", + "city_name": "Tumba", + "country_iso_code": "SE", + "country_name": "Sweden", + "region_name": "Stockholm", "location": { - "lon": -77.4728, - "lat": 39.0481 + "lon": 17.8167, + "lat": 59.2 } }, "cloud": { "provider": "aws" }, - "@timestamp": "2019-08-01T00:24:43.000Z", + "ecs": { + "version": "1.12.0" + }, "related": { "user": [ "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2" ], "ip": [ - "72.21.217.31" + "89.160.20.156" ] }, "http": { @@ -353,8 +373,8 @@ "user": { "id": "arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9" }, - "address": "72.21.217.31", - "ip": "72.21.217.31" + "address": "89.160.20.156", + "ip": "89.160.20.156" }, "tls": { "cipher": "ECDHE-RSA-AES128-SHA", @@ -363,8 +383,8 @@ }, "event": { "duration": 4000000, - "ingested": "2021-04-23T12:15:55.855317083Z", - "original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:43 +0000] 72.21.217.31 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 706992E2F3CC3C3D REST.GET.LOCATION - \"GET /test-s3-ks/?location\u0026aws-account=627959692251 HTTP/1.1\" 200 - 142 - 4 - \"-\" \"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation\" - cIN12KTrJwx+uTBZD+opZUPE4iGypi8oG/oXGPzFk9CMuHQGuEpmAeNELdtYKDxf2TDor25Nikg= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2", + "ingested": "2021-12-09T16:11:59.134208400Z", + "original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:43 +0000] 89.160.20.156 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 706992E2F3CC3C3D REST.GET.LOCATION - \"GET /test-s3-ks/?location\u0026aws-account=627959692251 HTTP/1.1\" 200 - 142 - 4 - \"-\" \"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation\" - cIN12KTrJwx+uTBZD+opZUPE4iGypi8oG/oXGPzFk9CMuHQGuEpmAeNELdtYKDxf2TDor25Nikg= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2", "kind": "event", "action": "REST.GET.LOCATION", "id": "706992E2F3CC3C3D", @@ -385,7 +405,7 @@ "host_id": "cIN12KTrJwx+uTBZD+opZUPE4iGypi8oG/oXGPzFk9CMuHQGuEpmAeNELdtYKDxf2TDor25Nikg=", "host_header": "s3.ap-southeast-1.amazonaws.com", "bucket": "test-s3-ks", - "remote_ip": "72.21.217.31", + "remote_ip": "89.160.20.156", "cipher_suite": "ECDHE-RSA-AES128-SHA", "http_status": 200, "total_time": 4, @@ -410,36 +430,41 @@ } }, { + "tags": [ + "preserve_original_event" + ], "geo": { "continent_name": "Europe", - "region_iso_code": "ES-TE", - "city_name": "Teruel", - "country_iso_code": "ES", - "country_name": "Spain", - "region_name": "Teruel", + "region_iso_code": "SE-AB", + "city_name": "Tumba", + "country_iso_code": "SE", + "country_name": "Sweden", + "region_name": "Stockholm", "location": { - "lon": -1.1065, - "lat": 40.3456 + "lon": 17.8167, + "lat": 59.2 } }, "cloud": { "provider": "aws" }, - "@timestamp": "2019-09-10T15:11:07.000Z", + "ecs": { + "version": "1.12.0" + }, "related": { "user": [ "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2" ], "ip": [ - "77.227.156.41" + "89.160.20.156" ] }, "client": { "user": { "id": "arn:aws:iam::123456:user/test@elastic.co" }, - "address": "77.227.156.41", - "ip": "77.227.156.41" + "address": "89.160.20.156", + "ip": "89.160.20.156" }, "http": { "response": { @@ -452,8 +477,8 @@ "version_protocol": "tls" }, "event": { - "ingested": "2021-04-23T12:15:55.855319015Z", - "original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 jsoriano-s3-test [10/Sep/2019:15:11:07 +0000] 77.227.156.41 arn:aws:iam::123456:user/test@elastic.co 8CD7A4A71E2E5C9E BATCH.DELETE.OBJECT jolokia-war-1.5.0.war - 204 - - 344017 - - - - - IeDW5I3wefFxU8iHOcAzi5qr+O+1bdRlcQ0AO2WGjFh7JwYM6qCoKq+1TrUshrXMlBxPFtg97Vk= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.eu-central-1.amazonaws.com TLSv1.2", + "ingested": "2021-12-09T16:11:59.134212900Z", + "original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 jsoriano-s3-test [10/Sep/2019:15:11:07 +0000] 89.160.20.156 arn:aws:iam::123456:user/test@elastic.co 8CD7A4A71E2E5C9E BATCH.DELETE.OBJECT jolokia-war-1.5.0.war - 204 - - 344017 - - - - - IeDW5I3wefFxU8iHOcAzi5qr+O+1bdRlcQ0AO2WGjFh7JwYM6qCoKq+1TrUshrXMlBxPFtg97Vk= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.eu-central-1.amazonaws.com TLSv1.2", "kind": "event", "action": "BATCH.DELETE.OBJECT", "id": "8CD7A4A71E2E5C9E", @@ -472,7 +497,7 @@ "host_id": "IeDW5I3wefFxU8iHOcAzi5qr+O+1bdRlcQ0AO2WGjFh7JwYM6qCoKq+1TrUshrXMlBxPFtg97Vk=", "host_header": "s3.eu-central-1.amazonaws.com", "bucket": "jsoriano-s3-test", - "remote_ip": "77.227.156.41", + "remote_ip": "89.160.20.156", "cipher_suite": "ECDHE-RSA-AES128-SHA", "http_status": 204, "bucket_owner": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2", @@ -484,36 +509,41 @@ } }, { + "tags": [ + "preserve_original_event" + ], "geo": { - "continent_name": "North America", - "region_iso_code": "US-CO", - "city_name": "Denver", - "country_iso_code": "US", - "country_name": "United States", - "region_name": "Colorado", + "continent_name": "Europe", + "region_iso_code": "SE-AB", + "city_name": "Tumba", + "country_iso_code": "SE", + "country_name": "Sweden", + "region_name": "Stockholm", "location": { - "lon": -105.0023, - "lat": 39.7044 + "lon": 17.8167, + "lat": 59.2 } }, "cloud": { "provider": "aws" }, - "@timestamp": "2019-09-19T17:06:39.000Z", + "ecs": { + "version": "1.12.0" + }, "related": { "user": [ "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2" ], "ip": [ - "174.29.206.152" + "89.160.20.156" ] }, "client": { "user": { "id": "arn:aws:iam::123456:user/test@elastic.co" }, - "address": "174.29.206.152", - "ip": "174.29.206.152" + "address": "89.160.20.156", + "ip": "89.160.20.156" }, "http": { "response": { @@ -526,8 +556,8 @@ "version_protocol": "tls" }, "event": { - "ingested": "2021-04-23T12:15:55.855320925Z", - "original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [19/Sep/2019:17:06:39 +0000] 174.29.206.152 arn:aws:iam::123456:user/test@elastic.co 6CE38F1312D32BDD BATCH.DELETE.OBJECT Screen+Shot+2019-09-09+at+9.08.44+AM.png - 204 - - 57138 - - - - - LwRa4w6DbuU48GKQiH3jDbjfTyLCbwasFBsdttugRQ+9lH4jK8lT91+HhGZKMYI3sPyKuQ9LvU0= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3-ap-southeast-1.amazonaws.com TLSv1.2", + "ingested": "2021-12-09T16:11:59.134217300Z", + "original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [19/Sep/2019:17:06:39 +0000] 89.160.20.156 arn:aws:iam::123456:user/test@elastic.co 6CE38F1312D32BDD BATCH.DELETE.OBJECT Screen+Shot+2019-09-09+at+9.08.44+AM.png - 204 - - 57138 - - - - - LwRa4w6DbuU48GKQiH3jDbjfTyLCbwasFBsdttugRQ+9lH4jK8lT91+HhGZKMYI3sPyKuQ9LvU0= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2", "kind": "event", "action": "BATCH.DELETE.OBJECT", "id": "6CE38F1312D32BDD", @@ -544,9 +574,9 @@ "signature_version": "SigV4", "authentication_type": "AuthHeader", "host_id": "LwRa4w6DbuU48GKQiH3jDbjfTyLCbwasFBsdttugRQ+9lH4jK8lT91+HhGZKMYI3sPyKuQ9LvU0=", - "host_header": "s3-ap-southeast-1.amazonaws.com", + "host_header": "s3.ap-southeast-1.amazonaws.com", "bucket": "test-s3-ks", - "remote_ip": "174.29.206.152", + "remote_ip": "89.160.20.156", "cipher_suite": "ECDHE-RSA-AES128-SHA", "http_status": 204, "bucket_owner": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2", @@ -556,6 +586,80 @@ "object_size": 57138 } } + }, + { + "url": { + "path": "/AWSLogs/000000000000/vpcflowlogs/us-gov-east-1/2021/07/13/000000000000_vpcflowlogs_us-gov-east-1_fl-0e7c13bf00cf15bfe_20210713T1855Z_f12aa632.log.gz", + "extension": "gz", + "original": "/AWSLogs/000000000000/vpcflowlogs/us-gov-east-1/2021/07/13/000000000000_vpcflowlogs_us-gov-east-1_fl-0e7c13bf00cf15bfe_20210713T1855Z_f12aa632.log.gz" + }, + "tags": [ + "preserve_original_event" + ], + "cloud": { + "provider": "aws" + }, + "ecs": { + "version": "1.12.0" + }, + "related": { + "user": [ + "67797214d75628047d9c76b18a78cded1a4b069b71f2a9d5a53649c38da8770b" + ] + }, + "http": { + "request": { + "method": "PUT" + }, + "version": "1.1", + "response": { + "status_code": 200 + } + }, + "client": { + "user": { + "id": "svc:delivery.logs.amazonaws.com" + } + }, + "tls": { + "cipher": "ECDHE-RSA-AES128-GCM-SHA256", + "version": "1.2", + "version_protocol": "tls" + }, + "event": { + "duration": 103000000, + "ingested": "2021-12-09T16:11:59.134221Z", + "original": "67797214d75628047d9c76b18a78cded1a4b069b71f2a9d5a53649c38da8770b flow-log-test [14/Jul/2021:18:57:31 +0000] - svc:delivery.logs.amazonaws.com MVGXZXEVN3IG9S24 REST.PUT.OBJECT AWSLogs/000000000000/vpcflowlogs/us-gov-east-1/2021/07/13/000000000000_vpcflowlogs_us-gov-east-1_fl-_20210713T1855Z_f12aa632.log.gz \"PUT /AWSLogs/000000000000/vpcflowlogs/us-gov-east-1/2021/07/13/000000000000_vpcflowlogs_us-gov-east-1_fl-0e7c13bf00cf15bfe_20210713T1855Z_f12aa632.log.gz HTTP/1.1\" 200 - - 773 103 13 \"-\" \"-\" - 02SxwfXpO5UysN0GsKGa3uGDQ6E/W7+Hwo/luRH8p1VEexULoe66RCM+nja0dEq2JqLrtgjocvVRRkVt4= SigV4 ECDHE-RSA-AES128-GCM-SHA256 AuthHeader flow-log-test.s3.us-gov-west-1.amazonaws.com TLSv1.2 -", + "kind": "event", + "action": "REST.PUT.OBJECT", + "id": "MVGXZXEVN3IG9S24", + "category": "web", + "type": [ + "access" + ], + "outcome": "success" + }, + "aws": { + "s3access": { + "requester": "svc:delivery.logs.amazonaws.com", + "tls_version": "TLSv1.2", + "signature_version": "SigV4", + "turn_around_time": 13, + "authentication_type": "AuthHeader", + "request_uri": "PUT /AWSLogs/000000000000/vpcflowlogs/us-gov-east-1/2021/07/13/000000000000_vpcflowlogs_us-gov-east-1_fl-0e7c13bf00cf15bfe_20210713T1855Z_f12aa632.log.gz HTTP/1.1", + "host_id": "02SxwfXpO5UysN0GsKGa3uGDQ6E/W7+Hwo/luRH8p1VEexULoe66RCM+nja0dEq2JqLrtgjocvVRRkVt4=", + "host_header": "flow-log-test.s3.us-gov-west-1.amazonaws.com", + "bucket": "flow-log-test", + "cipher_suite": "ECDHE-RSA-AES128-GCM-SHA256", + "http_status": 200, + "total_time": 103, + "bucket_owner": "67797214d75628047d9c76b18a78cded1a4b069b71f2a9d5a53649c38da8770b", + "operation": "REST.PUT.OBJECT", + "request_id": "MVGXZXEVN3IG9S24", + "key": "AWSLogs/000000000000/vpcflowlogs/us-gov-east-1/2021/07/13/000000000000_vpcflowlogs_us-gov-east-1_fl-_20210713T1855Z_f12aa632.log.gz", + "object_size": 773 + } + } } ] } \ No newline at end of file diff --git a/test/packages/aws/data_stream/elb_logs/agent/stream/s3.yml.hbs b/test/packages/aws/data_stream/s3access/agent/stream/aws-s3.yml.hbs similarity index 64% rename from test/packages/aws/data_stream/elb_logs/agent/stream/s3.yml.hbs rename to test/packages/aws/data_stream/s3access/agent/stream/aws-s3.yml.hbs index 6a00835778..ccf43bcddc 100644 --- a/test/packages/aws/data_stream/elb_logs/agent/stream/s3.yml.hbs +++ b/test/packages/aws/data_stream/s3access/agent/stream/aws-s3.yml.hbs @@ -11,6 +11,9 @@ visibility_timeout: {{visibility_timeout}} {{#if api_timeout}} api_timeout: {{api_timeout}} {{/if}} +{{#if max_number_of_messages}} +max_number_of_messages: {{max_number_of_messages}} +{{/if}} {{#if endpoint}} endpoint: {{endpoint}} {{/if}} @@ -29,8 +32,20 @@ role_arn: {{role_arn}} {{#if fips_enabled}} fips_enabled: {{fips_enabled}} {{/if}} +{{#if proxy_url }} +proxy_url: {{proxy_url}} +{{/if}} +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} processors: - - add_fields: - target: '' - fields: - ecs.version: 1.5.0 \ No newline at end of file +{{processors}} +{{/if}} \ No newline at end of file diff --git a/test/packages/aws/data_stream/s3access/agent/stream/log.yml.hbs b/test/packages/aws/data_stream/s3access/agent/stream/log.yml.hbs index 31a201bc45..de93a0f025 100644 --- a/test/packages/aws/data_stream/s3access/agent/stream/log.yml.hbs +++ b/test/packages/aws/data_stream/s3access/agent/stream/log.yml.hbs @@ -2,9 +2,18 @@ paths: {{#each paths as |path i|}} - {{path}} {{/each}} +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} exclude_files: [".gz$"] +{{#if processors}} processors: - - add_fields: - target: '' - fields: - ecs.version: 1.8.0 \ No newline at end of file +{{processors}} +{{/if}} \ No newline at end of file diff --git a/test/packages/aws/data_stream/s3access/agent/stream/s3.yml.hbs b/test/packages/aws/data_stream/s3access/agent/stream/s3.yml.hbs deleted file mode 100644 index cea7960b97..0000000000 --- a/test/packages/aws/data_stream/s3access/agent/stream/s3.yml.hbs +++ /dev/null @@ -1,36 +0,0 @@ -queue_url: {{queue_url}} -{{#if credential_profile_name}} -credential_profile_name: {{credential_profile_name}} -{{/if}} -{{#if shared_credential_file}} -shared_credential_file: {{shared_credential_file}} -{{/if}} -{{#if visibility_timeout}} -visibility_timeout: {{visibility_timeout}} -{{/if}} -{{#if api_timeout}} -api_timeout: {{api_timeout}} -{{/if}} -{{#if endpoint}} -endpoint: {{endpoint}} -{{/if}} -{{#if access_key_id}} -access_key_id: {{access_key_id}} -{{/if}} -{{#if secret_access_key}} -secret_access_key: {{secret_access_key}} -{{/if}} -{{#if session_token}} -session_token: {{session_token}} -{{/if}} -{{#if role_arn}} -role_arn: {{role_arn}} -{{/if}} -{{#if fips_enabled}} -fips_enabled: {{fips_enabled}} -{{/if}} -processors: - - add_fields: - target: '' - fields: - ecs.version: 1.8.0 \ No newline at end of file diff --git a/test/packages/aws/data_stream/s3access/elasticsearch/ingest_pipeline/default.yml b/test/packages/aws/data_stream/s3access/elasticsearch/ingest_pipeline/default.yml index 74a565e828..608a800390 100644 --- a/test/packages/aws/data_stream/s3access/elasticsearch/ingest_pipeline/default.yml +++ b/test/packages/aws/data_stream/s3access/elasticsearch/ingest_pipeline/default.yml @@ -2,21 +2,28 @@ description: "Pipeline for s3 server access logs" processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + - set: + field: ecs.version + value: '1.12.0' - set: field: event.category value: web - append: field: event.type value: access - - set: - field: event.ingested - value: '{{_ingest.timestamp}}' - - grok: + - rename: field: message + target_field: event.original + ignore_missing: true + - grok: + field: event.original patterns: - >- %{BASE16NUM:aws.s3access.bucket_owner} %{HOSTNAME:aws.s3access.bucket} \[%{HTTPDATE:_temp_.s3access_time}\] - %{IP:aws.s3access.remote_ip} (?:-|%{S3REQUESTER:aws.s3access.requester}) %{S3REQUESTID:aws.s3access.request_id} + (?:-|%{IP:aws.s3access.remote_ip}) (?:-|%{S3REQUESTER:aws.s3access.requester}) %{S3REQUESTID:aws.s3access.request_id} %{S3OPERATION:aws.s3access.operation} (?:-|%{S3KEY:aws.s3access.key}) (?:-|\"%{DATA:aws.s3access.request_uri}\") %{NUMBER:aws.s3access.http_status:long} (?:-|%{WORD:aws.s3access.error_code}) (?:-|%{NUMBER:aws.s3access.bytes_sent:long}) (?:-|%{NUMBER:aws.s3access.object_size:long}) (?:-|%{NUMBER:aws.s3access.total_time:long}) (?:-|%{NUMBER:aws.s3access.turn_around_time:long}) @@ -30,7 +37,6 @@ processors: S3KEY: "[a-zA-Z0-9\\/_\\.\\-%+]+" S3ID: "[a-zA-Z0-9\\/_\\.\\-%+=]+" S3VERSION: "[a-zA-Z0-9.]+" - - script: description: Drops null/empty values recursively lang: painless @@ -48,110 +54,89 @@ processors: return false; } drop(ctx); - - grok: field: aws.s3access.request_uri ignore_failure: true patterns: - - '%{NOTSPACE:http.request.method} %{NOTSPACE:url.original} [hH][tT][tT][pP]/%{NOTSPACE:http.version}' - - # - # Best-effort parse of url.original in the form /path?query" - # - - grok: - field: url.original - ignore_failure: true - patterns: - - '^%{ABS_PATH:url.path}(?:\?%{DATA:url.query})?$' - pattern_definitions: - ABS_PATH: '/[^?]*' + - '%{NOTSPACE:http.request.method} %{NOTSPACE:_temp_.url} [hH][tT][tT][pP]/%{NOTSPACE:http.version}' + - uri_parts: + field: _temp_.url + target_field: url + keep_original: true + if: ctx._temp_?.url != null - append: - if: "ctx?.aws?.s3access?.bucket_owner != null" field: related.user - value: "{{aws.s3access.bucket_owner}}" - + value: '{{aws.s3access.bucket_owner}}' + allow_duplicates: false + if: ctx?.aws?.s3access?.bucket_owner != null # # Parse the date included in s3 access logs # - date: - field: "_temp_.s3access_time" - target_field: "@timestamp" + field: _temp_.s3access_time' + target_field: '@timestamp' ignore_failure: true formats: - - "dd/MMM/yyyy:H:m:s Z" - + - dd/MMM/yyyy:H:m:s Z - set: field: client.ip - value: "{{aws.s3access.remote_ip}}" + value: '{{aws.s3access.remote_ip}}' ignore_empty_value: true - - append: - if: "ctx?.aws?.s3access?.remote_ip != null" field: related.ip - value: "{{aws.s3access.remote_ip}}" - + value: '{{aws.s3access.remote_ip}}' + allow_duplicates: false + if: ctx?.aws?.s3access?.remote_ip != null - set: field: client.address - value: "{{aws.s3access.remote_ip}}" + value: '{{aws.s3access.remote_ip}}' ignore_empty_value: true - - geoip: - if: "ctx?.aws?.s3access?.remote_ip != null" field: aws.s3access.remote_ip target_field: geo - + if: ctx?.aws?.s3access?.remote_ip != null - set: field: client.user.id - value: "{{aws.s3access.requester}}" + value: '{{aws.s3access.requester}}' ignore_empty_value: true - - set: field: event.id - value: "{{aws.s3access.request_id}}" + value: '{{aws.s3access.request_id}}' ignore_empty_value: true - - set: field: event.action - value: "{{aws.s3access.operation}}" + value: '{{aws.s3access.operation}}' ignore_empty_value: true - - set: field: http.response.status_code - value: "{{aws.s3access.http_status}}" + value: '{{aws.s3access.http_status}}' ignore_empty_value: true - - convert: - if: "ctx?.http?.response?.status_code != null" field: http.response.status_code type: long - + if: ctx?.http?.response?.status_code != null - set: - if: "ctx?.aws?.s3access?.error_code != null" field: event.outcome value: failure - + if: ctx?.aws?.s3access?.error_code != null - set: field: event.code - value: "{{aws.s3access.error_code}}" + value: '{{aws.s3access.error_code}}' ignore_empty_value: true - - set: - if: "ctx?.aws?.s3access?.error_code == null" field: event.outcome value: success - + if: ctx?.aws?.s3access?.error_code == null - convert: field: aws.s3access.bytes_sent target_field: http.response.body.bytes type: long ignore_failure: true - - convert: field: aws.s3access.total_time target_field: event.duration type: long ignore_failure: true - - script: lang: painless if: ctx.event?.duration != null @@ -159,21 +144,17 @@ processors: MS_TO_NS: 1000000 source: >- ctx.event.duration *= params.MS_TO_NS; - - set: field: http.request.referrer - value: "{{aws.s3access.referrer}}" + value: '{{aws.s3access.referrer}}' ignore_empty_value: true - - user_agent: - if: "ctx?.aws?.s3access?.user_agent != null" + if: ctx?.aws?.s3access?.user_agent != null field: aws.s3access.user_agent - - set: field: tls.cipher value: '{{aws.s3access.cipher_suite}}' ignore_empty_value: true - - script: lang: painless if: ctx.aws?.s3access?.tls_version != null @@ -184,30 +165,48 @@ processors: } ctx.tls.version = parts[1]; ctx.tls.version_protocol = parts[0] - - set: field: cloud.provider value: aws - - set: field: event.kind value: event - - # - # Save original message into event.original - # - - rename: - field: "message" - target_field: "event.original" - # # Remove temporary fields # - remove: field: _temp_ ignore_missing: true - + - script: + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx); + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - set: - field: "error.message" - value: "{{ _ingest.on_failure_message }}" + field: 'error.message' + value: '{{ _ingest.on_failure_message }}' diff --git a/test/packages/aws/data_stream/s3access/fields/base-fields.yml b/test/packages/aws/data_stream/s3access/fields/base-fields.yml index 7c798f4534..4d53b53c1c 100644 --- a/test/packages/aws/data_stream/s3access/fields/base-fields.yml +++ b/test/packages/aws/data_stream/s3access/fields/base-fields.yml @@ -10,3 +10,11 @@ - name: '@timestamp' type: date description: Event timestamp. +- name: event.module + type: constant_keyword + description: Event module + value: aws +- name: event.dataset + type: constant_keyword + description: Event dataset + value: aws.s3access diff --git a/test/packages/aws/data_stream/s3access/fields/ecs.yml b/test/packages/aws/data_stream/s3access/fields/ecs.yml index 0d3655b8c1..ce6a6aac98 100644 --- a/test/packages/aws/data_stream/s3access/fields/ecs.yml +++ b/test/packages/aws/data_stream/s3access/fields/ecs.yml @@ -1,131 +1,88 @@ -- name: related.user - type: keyword - description: All the user names seen on your event. -- name: related.ip - type: ip - description: All of the IPs seen on your event. -- name: client.ip - type: ip - description: IP address of the client. -- name: client.address - type: keyword - description: Some event client addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the .address field. -- name: client.user.id - type: keyword - description: Unique identifiers of the user. -- name: event.id - type: keyword - description: Unique ID to describe the event. -- name: event.action - type: keyword - description: The action captured by the event. -- name: event.outcome - type: keyword - description: This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. -- name: event.code - type: keyword - description: Identification code for this event, if one exists. -- name: event.duration - type: long - description: Duration of the event in nanoseconds. -- name: http - title: HTTP - type: group - fields: - - name: request.method - type: keyword - ignore_above: 1024 - description: 'HTTP request method.' - - name: request.referrer - type: keyword - ignore_above: 1024 - description: Referrer for this HTTP request. - - name: response.body.bytes - type: long - format: bytes - description: Size in bytes of the response body. - - name: response.status_code - type: long - description: HTTP response status code. - - name: version - type: keyword - ignore_above: 1024 - description: HTTP version. -- name: url - title: URL - type: group - fields: - - name: original - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: 'Unmodified original url as seen in the event source.' - - name: path - type: keyword - ignore_above: 1024 - description: Path of the request, such as "/search". - - name: query - type: keyword - ignore_above: 1024 - description: 'The query field describes the query string of the request, such as "q=elasticsearch".' -- name: tls.cipher - type: keyword - description: String indicating the cipher used during the current connection. -- name: tls.version - type: keyword - description: Numeric part of the version parsed from the original string. -- name: tls.version_protocol - type: keyword - description: Normalized lowercase protocol name parsed from original string. -- name: cloud.provider - type: keyword - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. -- name: event.kind - type: keyword - description: Event kind (e.g. event, alert, metric, state, pipeline_error, signal) -- name: geo.city_name - type: keyword - description: City name. -- name: geo.country_name - type: keyword - description: Country name. -- name: geo.continent_name - type: keyword - description: Name of the continent. -- name: geo.country_iso_code - type: keyword - description: Country ISO code. -- name: geo.location +- external: ecs + name: client.address +- external: ecs + name: client.ip +- external: ecs + name: client.user.id +- external: ecs + name: cloud.provider +- external: ecs + name: ecs.version +- external: ecs + name: error.message +- external: ecs + name: event.action +- external: ecs + name: event.code +- external: ecs + name: event.duration +- external: ecs + name: event.id +- external: ecs + name: event.kind +- external: ecs + name: event.outcome +- external: ecs + name: geo.city_name +- external: ecs + name: geo.continent_name +- external: ecs + name: geo.country_iso_code +- external: ecs + name: geo.country_name +- description: Longitude and latitude. + level: core + name: geo.location type: geo_point - description: Longitude and latitude. -- name: geo.region_iso_code - type: keyword - description: Region ISO code. -- name: geo.region_name - type: keyword - description: Region name. -- name: user_agent.device.name - type: keyword - description: Name of the device. -- name: user_agent.name - type: keyword - description: Name of the user agent. -- name: user_agent.original - type: keyword - description: Unparsed user_agent string. -- name: user_agent.os.full - type: keyword - description: Operating system name, including the version or code name. -- name: user_agent.os.name - type: keyword - description: Operating system name, without the version. -- name: user_agent.os.version - type: keyword - description: Operating system version as a raw string. -- name: user_agent.version - type: keyword - description: Version of the user agent. +- external: ecs + name: geo.region_iso_code +- external: ecs + name: geo.region_name +- external: ecs + name: http.request.method +- external: ecs + name: http.request.referrer +- external: ecs + name: http.response.body.bytes +- external: ecs + name: http.response.status_code +- external: ecs + name: http.version +- external: ecs + name: related.ip +- external: ecs + name: related.user +- external: ecs + name: tags +- external: ecs + name: tls.cipher +- external: ecs + name: tls.version +- external: ecs + name: tls.version_protocol +- external: ecs + name: url.domain +- external: ecs + name: url.extension +- external: ecs + name: url.original +- external: ecs + name: url.path +- external: ecs + name: url.query +- external: ecs + name: url.scheme +- external: ecs + name: user_agent.device.name +- external: ecs + name: user_agent.name +- external: ecs + name: user_agent.original +- external: ecs + name: user_agent.os.full +- external: ecs + name: user_agent.os.name +- external: ecs + name: user_agent.os.version +- external: ecs + name: user_agent.version diff --git a/test/packages/aws/data_stream/s3access/manifest.yml b/test/packages/aws/data_stream/s3access/manifest.yml index 648a1d7f6f..b88c6ba490 100644 --- a/test/packages/aws/data_stream/s3access/manifest.yml +++ b/test/packages/aws/data_stream/s3access/manifest.yml @@ -1,12 +1,25 @@ title: AWS s3access logs -release: beta type: logs streams: - - input: s3 - template_path: s3.yml.hbs + - input: aws-s3 + template_path: aws-s3.yml.hbs title: AWS s3access logs description: Collect AWS s3access logs using s3 input vars: + - name: visibility_timeout + type: text + title: Visibility Timeout + multi: false + required: false + show_user: false + description: The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours. + - name: api_timeout + type: text + title: API Timeout + multi: false + required: false + show_user: false + description: The maximum duration of AWS API can take. The maximum is half of the visibility timeout value. - name: queue_url type: text title: Queue URL @@ -22,3 +35,36 @@ streams: required: false show_user: false description: Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - aws-s3access + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: max_number_of_messages + type: integer + title: Maximum Concurrent SQS Messages + description: The maximum number of SQS messages that can be inflight at any time. + default: 5 + required: false + show_user: false diff --git a/test/packages/aws/data_stream/s3access/sample_event.json b/test/packages/aws/data_stream/s3access/sample_event.json new file mode 100644 index 0000000000..87826bf8af --- /dev/null +++ b/test/packages/aws/data_stream/s3access/sample_event.json @@ -0,0 +1,113 @@ +{ + "@timestamp": "2021-11-26T14:44:27.652Z", + "data_stream": { + "namespace": "default", + "type": "logs", + "dataset": "aws.s3access" + }, + "url": { + "path": "/test-s3-ks/", + "original": "/test-s3-ks/?location\u0026aws-account=627959692251", + "query": "location\u0026aws-account=627959692251" + }, + "tags": [ + "preserve_original_event" + ], + "geo": { + "continent_name": "North America", + "region_iso_code": "US-VA", + "city_name": "Ashburn", + "country_iso_code": "US", + "country_name": "United States", + "region_name": "Virginia", + "location": { + "lon": -77.4728, + "lat": 39.0481 + } + }, + "cloud": { + "provider": "aws" + }, + "ecs": { + "version": "1.12.0" + }, + "related": { + "user": [ + "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2" + ], + "ip": [ + "72.21.217.31" + ] + }, + "http": { + "request": { + "method": "GET" + }, + "version": "1.1", + "response": { + "body": { + "bytes": 142 + }, + "status_code": 200 + } + }, + "client": { + "user": { + "id": "arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9" + }, + "address": "72.21.217.31", + "ip": "72.21.217.31" + }, + "tls": { + "cipher": "ECDHE-RSA-AES128-SHA", + "version": "1.2", + "version_protocol": "tls" + }, + "event": { + "duration": 17000000, + "ingested": "2021-07-19T21:47:05.259665700Z", + "original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:41 +0000] 72.21.217.31 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 44EE8651683CB4DA REST.GET.LOCATION - \"GET /test-s3-ks/?location\u0026aws-account=627959692251 HTTP/1.1\" 200 - 142 - 17 - \"-\" \"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation\" - BsCfJedfuSnds2QFoxi+E/O7M6OEWzJnw4dUaes/2hyA363sONRJKzB7EOY+Bt9DTHYUn+HoHxI= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2", + "kind": "event", + "action": "REST.GET.LOCATION", + "id": "44EE8651683CB4DA", + "category": "web", + "type": [ + "access" + ], + "outcome": "success" + }, + "aws": { + "s3access": { + "requester": "arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9", + "tls_version": "TLSv1.2", + "signature_version": "SigV4", + "bytes_sent": 142, + "authentication_type": "AuthHeader", + "request_uri": "GET /test-s3-ks/?location\u0026aws-account=627959692251 HTTP/1.1", + "host_id": "BsCfJedfuSnds2QFoxi+E/O7M6OEWzJnw4dUaes/2hyA363sONRJKzB7EOY+Bt9DTHYUn+HoHxI=", + "host_header": "s3.ap-southeast-1.amazonaws.com", + "bucket": "test-s3-ks", + "remote_ip": "72.21.217.31", + "cipher_suite": "ECDHE-RSA-AES128-SHA", + "http_status": 200, + "total_time": 17, + "bucket_owner": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2", + "operation": "REST.GET.LOCATION", + "request_id": "44EE8651683CB4DA", + "user_agent": "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation" + } + }, + "user_agent": { + "name": "aws-sdk-java", + "original": "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation", + "os": { + "name": "Linux", + "version": "4.9.137", + "full": "Linux 4.9.137" + }, + "device": { + "name": "Other" + }, + "version": "1.11.590" + } +} \ No newline at end of file diff --git a/test/packages/aws/data_stream/sns/agent/stream/stream.yml.hbs b/test/packages/aws/data_stream/sns/agent/stream/stream.yml.hbs index 6c56e029fb..9a12668207 100644 --- a/test/packages/aws/data_stream/sns/agent/stream/stream.yml.hbs +++ b/test/packages/aws/data_stream/sns/agent/stream/stream.yml.hbs @@ -29,4 +29,7 @@ latency: {{latency}} {{/if}} {{#if tags_filter}} tags_filter: {{tags_filter}} +{{/if}} +{{#if proxy_url }} +proxy_url: {{proxy_url}} {{/if}} \ No newline at end of file diff --git a/test/packages/aws/data_stream/sns/fields/base-fields.yml b/test/packages/aws/data_stream/sns/fields/base-fields.yml index 7c798f4534..17fbf36279 100644 --- a/test/packages/aws/data_stream/sns/fields/base-fields.yml +++ b/test/packages/aws/data_stream/sns/fields/base-fields.yml @@ -10,3 +10,11 @@ - name: '@timestamp' type: date description: Event timestamp. +- name: event.module + type: constant_keyword + description: Event module + value: aws +- name: event.dataset + type: constant_keyword + description: Event dataset + value: aws.sns diff --git a/test/packages/aws/data_stream/sns/fields/ecs.yml b/test/packages/aws/data_stream/sns/fields/ecs.yml index a02d7269c5..83e3f6f122 100644 --- a/test/packages/aws/data_stream/sns/fields/ecs.yml +++ b/test/packages/aws/data_stream/sns/fields/ecs.yml @@ -1,53 +1,24 @@ -- name: cloud - title: Cloud - group: 2 - type: group - footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' - fields: - - name: account.id - level: extended - type: keyword - description: |- - The cloud account or organization id used to identify different entities in a multi-tenant environment. - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. - ignore_above: 1024 - - name: account.name - level: extended - type: keyword - description: |- - The cloud account name or alias used to identify different entities in a multi-tenant environment. - Examples: AWS account name, Google Cloud ORG display name. - ignore_above: 1024 - - name: availability_zone - level: extended - type: keyword - description: Availability zone in which this host is running. - ignore_above: 1024 - - name: instance.id - level: extended - type: keyword - description: Instance ID of the host machine. - ignore_above: 1024 - - name: machine.type - level: extended - type: keyword - description: Machine type of the host machine. - ignore_above: 1024 - - name: provider - level: extended - type: keyword - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - ignore_above: 1024 - - name: region - level: extended - type: keyword - description: Region in which this host is running. - ignore_above: 1024 -- name: ecs.version - type: keyword - description: ECS version this event conforms to. - example: 1.0.0 - ignore_above: 1024 -- name: service.type - type: keyword - description: Service type +- external: ecs + name: cloud +- external: ecs + name: cloud.account.id +- external: ecs + name: cloud.account.name +- external: ecs + name: cloud.availability_zone +- external: ecs + name: cloud.instance.id +- external: ecs + name: cloud.machine.type +- external: ecs + name: cloud.provider +- external: ecs + name: cloud.region +- external: ecs + name: ecs.version +- external: ecs + name: error +- external: ecs + name: error.message +- external: ecs + name: service.type diff --git a/test/packages/aws/data_stream/sns/manifest.yml b/test/packages/aws/data_stream/sns/manifest.yml index 806a5e416b..9ec9ad0daa 100644 --- a/test/packages/aws/data_stream/sns/manifest.yml +++ b/test/packages/aws/data_stream/sns/manifest.yml @@ -1,5 +1,4 @@ title: AWS SNS metrics -release: beta type: metrics streams: - input: aws/metrics diff --git a/test/packages/aws/data_stream/sqs/agent/stream/stream.yml.hbs b/test/packages/aws/data_stream/sqs/agent/stream/stream.yml.hbs index 2e9f1a2d15..0cf05645ac 100644 --- a/test/packages/aws/data_stream/sqs/agent/stream/stream.yml.hbs +++ b/test/packages/aws/data_stream/sqs/agent/stream/stream.yml.hbs @@ -26,4 +26,7 @@ regions: {{/if}} {{#if latency}} latency: {{latency}} +{{/if}} +{{#if proxy_url }} +proxy_url: {{proxy_url}} {{/if}} \ No newline at end of file diff --git a/test/packages/aws/data_stream/sqs/fields/base-fields.yml b/test/packages/aws/data_stream/sqs/fields/base-fields.yml index 7c798f4534..c39e5d890e 100644 --- a/test/packages/aws/data_stream/sqs/fields/base-fields.yml +++ b/test/packages/aws/data_stream/sqs/fields/base-fields.yml @@ -10,3 +10,11 @@ - name: '@timestamp' type: date description: Event timestamp. +- name: event.module + type: constant_keyword + description: Event module + value: aws +- name: event.dataset + type: constant_keyword + description: Event dataset + value: aws.sqs diff --git a/test/packages/aws/data_stream/sqs/fields/ecs.yml b/test/packages/aws/data_stream/sqs/fields/ecs.yml index a02d7269c5..83e3f6f122 100644 --- a/test/packages/aws/data_stream/sqs/fields/ecs.yml +++ b/test/packages/aws/data_stream/sqs/fields/ecs.yml @@ -1,53 +1,24 @@ -- name: cloud - title: Cloud - group: 2 - type: group - footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' - fields: - - name: account.id - level: extended - type: keyword - description: |- - The cloud account or organization id used to identify different entities in a multi-tenant environment. - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. - ignore_above: 1024 - - name: account.name - level: extended - type: keyword - description: |- - The cloud account name or alias used to identify different entities in a multi-tenant environment. - Examples: AWS account name, Google Cloud ORG display name. - ignore_above: 1024 - - name: availability_zone - level: extended - type: keyword - description: Availability zone in which this host is running. - ignore_above: 1024 - - name: instance.id - level: extended - type: keyword - description: Instance ID of the host machine. - ignore_above: 1024 - - name: machine.type - level: extended - type: keyword - description: Machine type of the host machine. - ignore_above: 1024 - - name: provider - level: extended - type: keyword - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - ignore_above: 1024 - - name: region - level: extended - type: keyword - description: Region in which this host is running. - ignore_above: 1024 -- name: ecs.version - type: keyword - description: ECS version this event conforms to. - example: 1.0.0 - ignore_above: 1024 -- name: service.type - type: keyword - description: Service type +- external: ecs + name: cloud +- external: ecs + name: cloud.account.id +- external: ecs + name: cloud.account.name +- external: ecs + name: cloud.availability_zone +- external: ecs + name: cloud.instance.id +- external: ecs + name: cloud.machine.type +- external: ecs + name: cloud.provider +- external: ecs + name: cloud.region +- external: ecs + name: ecs.version +- external: ecs + name: error +- external: ecs + name: error.message +- external: ecs + name: service.type diff --git a/test/packages/aws/data_stream/sqs/fields/fields.yml b/test/packages/aws/data_stream/sqs/fields/fields.yml index a6f2304201..c1e5e0241e 100644 --- a/test/packages/aws/data_stream/sqs/fields/fields.yml +++ b/test/packages/aws/data_stream/sqs/fields/fields.yml @@ -52,3 +52,9 @@ type: keyword description: | SQS queue name + - name: cloudwatch + type: group + fields: + - name: namespace + type: keyword + description: The namespace specified when query cloudwatch api. diff --git a/test/packages/aws/data_stream/sqs/manifest.yml b/test/packages/aws/data_stream/sqs/manifest.yml index b1a57a9faf..864d57bf98 100644 --- a/test/packages/aws/data_stream/sqs/manifest.yml +++ b/test/packages/aws/data_stream/sqs/manifest.yml @@ -1,5 +1,4 @@ title: AWS SQS metrics -release: beta type: metrics streams: - input: aws/metrics diff --git a/test/packages/aws/data_stream/transitgateway/agent/stream/stream.yml.hbs b/test/packages/aws/data_stream/transitgateway/agent/stream/stream.yml.hbs index b5530f1b2c..4a574dfdca 100644 --- a/test/packages/aws/data_stream/transitgateway/agent/stream/stream.yml.hbs +++ b/test/packages/aws/data_stream/transitgateway/agent/stream/stream.yml.hbs @@ -29,4 +29,7 @@ latency: {{latency}} {{/if}} {{#if tags_filter}} tags_filter: {{tags_filter}} +{{/if}} +{{#if proxy_url }} +proxy_url: {{proxy_url}} {{/if}} \ No newline at end of file diff --git a/test/packages/aws/data_stream/transitgateway/fields/base-fields.yml b/test/packages/aws/data_stream/transitgateway/fields/base-fields.yml index 7c798f4534..291688d695 100644 --- a/test/packages/aws/data_stream/transitgateway/fields/base-fields.yml +++ b/test/packages/aws/data_stream/transitgateway/fields/base-fields.yml @@ -10,3 +10,11 @@ - name: '@timestamp' type: date description: Event timestamp. +- name: event.module + type: constant_keyword + description: Event module + value: aws +- name: event.dataset + type: constant_keyword + description: Event dataset + value: aws.transitgateway diff --git a/test/packages/aws/data_stream/transitgateway/fields/ecs.yml b/test/packages/aws/data_stream/transitgateway/fields/ecs.yml index a02d7269c5..83e3f6f122 100644 --- a/test/packages/aws/data_stream/transitgateway/fields/ecs.yml +++ b/test/packages/aws/data_stream/transitgateway/fields/ecs.yml @@ -1,53 +1,24 @@ -- name: cloud - title: Cloud - group: 2 - type: group - footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' - fields: - - name: account.id - level: extended - type: keyword - description: |- - The cloud account or organization id used to identify different entities in a multi-tenant environment. - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. - ignore_above: 1024 - - name: account.name - level: extended - type: keyword - description: |- - The cloud account name or alias used to identify different entities in a multi-tenant environment. - Examples: AWS account name, Google Cloud ORG display name. - ignore_above: 1024 - - name: availability_zone - level: extended - type: keyword - description: Availability zone in which this host is running. - ignore_above: 1024 - - name: instance.id - level: extended - type: keyword - description: Instance ID of the host machine. - ignore_above: 1024 - - name: machine.type - level: extended - type: keyword - description: Machine type of the host machine. - ignore_above: 1024 - - name: provider - level: extended - type: keyword - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - ignore_above: 1024 - - name: region - level: extended - type: keyword - description: Region in which this host is running. - ignore_above: 1024 -- name: ecs.version - type: keyword - description: ECS version this event conforms to. - example: 1.0.0 - ignore_above: 1024 -- name: service.type - type: keyword - description: Service type +- external: ecs + name: cloud +- external: ecs + name: cloud.account.id +- external: ecs + name: cloud.account.name +- external: ecs + name: cloud.availability_zone +- external: ecs + name: cloud.instance.id +- external: ecs + name: cloud.machine.type +- external: ecs + name: cloud.provider +- external: ecs + name: cloud.region +- external: ecs + name: ecs.version +- external: ecs + name: error +- external: ecs + name: error.message +- external: ecs + name: service.type diff --git a/test/packages/aws/data_stream/transitgateway/manifest.yml b/test/packages/aws/data_stream/transitgateway/manifest.yml index 36ed6e401b..d433369d08 100644 --- a/test/packages/aws/data_stream/transitgateway/manifest.yml +++ b/test/packages/aws/data_stream/transitgateway/manifest.yml @@ -1,5 +1,4 @@ title: AWS Transit Gateway metrics -release: beta type: metrics streams: - input: aws/metrics diff --git a/test/packages/aws/data_stream/usage/agent/stream/stream.yml.hbs b/test/packages/aws/data_stream/usage/agent/stream/stream.yml.hbs index 24c082cd4d..5298d8d404 100644 --- a/test/packages/aws/data_stream/usage/agent/stream/stream.yml.hbs +++ b/test/packages/aws/data_stream/usage/agent/stream/stream.yml.hbs @@ -29,4 +29,7 @@ latency: {{latency}} {{/if}} {{#if tags_filter}} tags_filter: {{tags_filter}} +{{/if}} +{{#if proxy_url }} +proxy_url: {{proxy_url}} {{/if}} \ No newline at end of file diff --git a/test/packages/aws/data_stream/usage/fields/base-fields.yml b/test/packages/aws/data_stream/usage/fields/base-fields.yml index 7c798f4534..d774fe1ffb 100644 --- a/test/packages/aws/data_stream/usage/fields/base-fields.yml +++ b/test/packages/aws/data_stream/usage/fields/base-fields.yml @@ -10,3 +10,11 @@ - name: '@timestamp' type: date description: Event timestamp. +- name: event.module + type: constant_keyword + description: Event module + value: aws +- name: event.dataset + type: constant_keyword + description: Event dataset + value: aws.usage diff --git a/test/packages/aws/data_stream/usage/fields/ecs.yml b/test/packages/aws/data_stream/usage/fields/ecs.yml index a02d7269c5..83e3f6f122 100644 --- a/test/packages/aws/data_stream/usage/fields/ecs.yml +++ b/test/packages/aws/data_stream/usage/fields/ecs.yml @@ -1,53 +1,24 @@ -- name: cloud - title: Cloud - group: 2 - type: group - footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' - fields: - - name: account.id - level: extended - type: keyword - description: |- - The cloud account or organization id used to identify different entities in a multi-tenant environment. - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. - ignore_above: 1024 - - name: account.name - level: extended - type: keyword - description: |- - The cloud account name or alias used to identify different entities in a multi-tenant environment. - Examples: AWS account name, Google Cloud ORG display name. - ignore_above: 1024 - - name: availability_zone - level: extended - type: keyword - description: Availability zone in which this host is running. - ignore_above: 1024 - - name: instance.id - level: extended - type: keyword - description: Instance ID of the host machine. - ignore_above: 1024 - - name: machine.type - level: extended - type: keyword - description: Machine type of the host machine. - ignore_above: 1024 - - name: provider - level: extended - type: keyword - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - ignore_above: 1024 - - name: region - level: extended - type: keyword - description: Region in which this host is running. - ignore_above: 1024 -- name: ecs.version - type: keyword - description: ECS version this event conforms to. - example: 1.0.0 - ignore_above: 1024 -- name: service.type - type: keyword - description: Service type +- external: ecs + name: cloud +- external: ecs + name: cloud.account.id +- external: ecs + name: cloud.account.name +- external: ecs + name: cloud.availability_zone +- external: ecs + name: cloud.instance.id +- external: ecs + name: cloud.machine.type +- external: ecs + name: cloud.provider +- external: ecs + name: cloud.region +- external: ecs + name: ecs.version +- external: ecs + name: error +- external: ecs + name: error.message +- external: ecs + name: service.type diff --git a/test/packages/aws/data_stream/usage/manifest.yml b/test/packages/aws/data_stream/usage/manifest.yml index ca2c781a65..bed34afa7c 100644 --- a/test/packages/aws/data_stream/usage/manifest.yml +++ b/test/packages/aws/data_stream/usage/manifest.yml @@ -1,5 +1,4 @@ title: AWS usage metrics -release: beta type: metrics streams: - input: aws/metrics diff --git a/test/packages/aws/data_stream/vpcflow/_dev/test/pipeline/test-common-config.yml b/test/packages/aws/data_stream/vpcflow/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 0000000000..5622947e4b --- /dev/null +++ b/test/packages/aws/data_stream/vpcflow/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,5 @@ +dynamic_fields: + event.ingested: ".*" +fields: + tags: + - preserve_original_event diff --git a/test/packages/aws/data_stream/vpcflow/_dev/test/pipeline/test-extra-samples.log b/test/packages/aws/data_stream/vpcflow/_dev/test/pipeline/test-extra-samples.log new file mode 100644 index 0000000000..808ade66dc --- /dev/null +++ b/test/packages/aws/data_stream/vpcflow/_dev/test/pipeline/test-extra-samples.log @@ -0,0 +1,7 @@ +2 123456789010 eni-1235b8ca123456789 2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6 2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6 34892 22 6 54 8855 1477913708 1477913820 ACCEPT OK +2 123456789010 eni-1235b8ca123456789 - - - - - - - 1431280876 1431280934 - NODATA +2 123456789010 eni-89.160.20.1561aaaaaaaaa - - - - - - - 1431280876 1431280934 - SKIPDATA +2 123456789010 eni-1235b8ca123456789 89.160.20.156 89.160.20.156 20641 22 6 20 4249 1418530010 1418530070 ACCEPT OK +2 123456789010 eni-1235b8ca123456789 89.160.20.156 89.160.20.156 49761 3389 6 20 4249 1418530010 1418530070 REJECT OK +2 123456789010 eni-1235b8ca123456789 89.160.20.156 172.31.16.139 0 0 1 4 336 1432917027 1432917142 ACCEPT OK +2 123456789010 eni-1235b8ca123456789 172.31.16.139 89.160.20.156 0 0 1 4 336 1432917094 1432917142 REJECT OK \ No newline at end of file diff --git a/test/packages/aws/data_stream/vpcflow/_dev/test/pipeline/test-extra-samples.log-expected.json b/test/packages/aws/data_stream/vpcflow/_dev/test/pipeline/test-extra-samples.log-expected.json new file mode 100644 index 0000000000..77a00b72a0 --- /dev/null +++ b/test/packages/aws/data_stream/vpcflow/_dev/test/pipeline/test-extra-samples.log-expected.json @@ -0,0 +1,504 @@ +{ + "expected": [ + { + "destination": { + "geo": { + "continent_name": "Europe", + "country_name": "Denmark", + "location": { + "lon": 10.0, + "lat": 56.0 + }, + "country_iso_code": "DK" + }, + "as": { + "number": 62121, + "organization": { + "name": "Christian Ebsen ApS" + } + }, + "address": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6", + "port": 22, + "ip": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6" + }, + "source": { + "geo": { + "continent_name": "Europe", + "country_name": "Denmark", + "location": { + "lon": 10.0, + "lat": 56.0 + }, + "country_iso_code": "DK" + }, + "as": { + "number": 62121, + "organization": { + "name": "Christian Ebsen ApS" + } + }, + "address": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6", + "port": 34892, + "bytes": 8855, + "ip": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6", + "packets": 54 + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "community_id": "1:3piNHoW0DjbrWkF//BeRomCaOZQ=", + "transport": "tcp", + "type": "ipv6", + "bytes": 8855, + "iana_number": "6", + "packets": 54 + }, + "cloud": { + "provider": "aws", + "account": { + "id": "123456789010" + } + }, + "@timestamp": "2016-10-31T11:37:00.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "ip": [ + "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6", + "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6" + ] + }, + "event": { + "ingested": "2021-12-09T16:12:00.503382700Z", + "original": "2 123456789010 eni-1235b8ca123456789 2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6 2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6 34892 22 6 54 8855 1477913708 1477913820 ACCEPT OK", + "kind": "event", + "start": "2016-10-31T11:35:08.000Z", + "end": "2016-10-31T11:37:00.000Z", + "type": "flow", + "category": "network_traffic", + "outcome": "allow" + }, + "aws": { + "vpcflow": { + "action": "ACCEPT", + "account_id": "123456789010", + "log_status": "OK", + "interface_id": "eni-1235b8ca123456789", + "version": "2" + } + } + }, + { + "cloud": { + "provider": "aws", + "account": { + "id": "123456789010" + } + }, + "@timestamp": "2015-05-10T18:02:14.000Z", + "ecs": { + "version": "1.12.0" + }, + "event": { + "ingested": "2021-12-09T16:12:00.503391600Z", + "original": "2 123456789010 eni-1235b8ca123456789 - - - - - - - 1431280876 1431280934 - NODATA", + "kind": "event", + "start": "2015-05-10T18:01:16.000Z", + "end": "2015-05-10T18:02:14.000Z", + "type": "flow", + "category": "network_traffic" + }, + "aws": { + "vpcflow": { + "account_id": "123456789010", + "log_status": "NODATA", + "interface_id": "eni-1235b8ca123456789", + "version": "2" + } + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "cloud": { + "provider": "aws", + "account": { + "id": "123456789010" + } + }, + "@timestamp": "2015-05-10T18:02:14.000Z", + "ecs": { + "version": "1.12.0" + }, + "event": { + "ingested": "2021-12-09T16:12:00.503397800Z", + "original": "2 123456789010 eni-89.160.20.1561aaaaaaaaa - - - - - - - 1431280876 1431280934 - SKIPDATA", + "kind": "event", + "start": "2015-05-10T18:01:16.000Z", + "end": "2015-05-10T18:02:14.000Z", + "type": "flow", + "category": "network_traffic" + }, + "aws": { + "vpcflow": { + "account_id": "123456789010", + "log_status": "SKIPDATA", + "interface_id": "eni-89.160.20.1561aaaaaaaaa", + "version": "2" + } + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "destination": { + "geo": { + "continent_name": "Europe", + "region_iso_code": "SE-AB", + "city_name": "Tumba", + "country_iso_code": "SE", + "country_name": "Sweden", + "region_name": "Stockholm", + "location": { + "lon": 17.8167, + "lat": 59.2 + } + }, + "as": { + "number": 29518, + "organization": { + "name": "Bredband2 AB" + } + }, + "address": "89.160.20.156", + "port": 22, + "ip": "89.160.20.156" + }, + "source": { + "geo": { + "continent_name": "Europe", + "region_iso_code": "SE-AB", + "city_name": "Tumba", + "country_iso_code": "SE", + "country_name": "Sweden", + "region_name": "Stockholm", + "location": { + "lon": 17.8167, + "lat": 59.2 + } + }, + "as": { + "number": 29518, + "organization": { + "name": "Bredband2 AB" + } + }, + "address": "89.160.20.156", + "port": 20641, + "bytes": 4249, + "ip": "89.160.20.156", + "packets": 20 + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "community_id": "1:CEGBlG6oEeW2Y5LLdr9GONITz00=", + "transport": "tcp", + "type": "ipv4", + "bytes": 4249, + "iana_number": "6", + "packets": 20 + }, + "cloud": { + "provider": "aws", + "account": { + "id": "123456789010" + } + }, + "@timestamp": "2014-12-14T04:07:50.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "ip": [ + "89.160.20.156", + "89.160.20.156" + ] + }, + "event": { + "ingested": "2021-12-09T16:12:00.503403700Z", + "original": "2 123456789010 eni-1235b8ca123456789 89.160.20.156 89.160.20.156 20641 22 6 20 4249 1418530010 1418530070 ACCEPT OK", + "kind": "event", + "start": "2014-12-14T04:06:50.000Z", + "end": "2014-12-14T04:07:50.000Z", + "type": "flow", + "category": "network_traffic", + "outcome": "allow" + }, + "aws": { + "vpcflow": { + "action": "ACCEPT", + "account_id": "123456789010", + "log_status": "OK", + "interface_id": "eni-1235b8ca123456789", + "version": "2" + } + } + }, + { + "destination": { + "geo": { + "continent_name": "Europe", + "region_iso_code": "SE-AB", + "city_name": "Tumba", + "country_iso_code": "SE", + "country_name": "Sweden", + "region_name": "Stockholm", + "location": { + "lon": 17.8167, + "lat": 59.2 + } + }, + "as": { + "number": 29518, + "organization": { + "name": "Bredband2 AB" + } + }, + "address": "89.160.20.156", + "port": 3389, + "ip": "89.160.20.156" + }, + "source": { + "geo": { + "continent_name": "Europe", + "region_iso_code": "SE-AB", + "city_name": "Tumba", + "country_iso_code": "SE", + "country_name": "Sweden", + "region_name": "Stockholm", + "location": { + "lon": 17.8167, + "lat": 59.2 + } + }, + "as": { + "number": 29518, + "organization": { + "name": "Bredband2 AB" + } + }, + "address": "89.160.20.156", + "port": 49761, + "bytes": 4249, + "ip": "89.160.20.156", + "packets": 20 + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "community_id": "1:va8LK/uEqYpj4NoZ9/5WRLio5rs=", + "transport": "tcp", + "type": "ipv4", + "bytes": 4249, + "iana_number": "6", + "packets": 20 + }, + "cloud": { + "provider": "aws", + "account": { + "id": "123456789010" + } + }, + "@timestamp": "2014-12-14T04:07:50.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "ip": [ + "89.160.20.156", + "89.160.20.156" + ] + }, + "event": { + "ingested": "2021-12-09T16:12:00.503409900Z", + "original": "2 123456789010 eni-1235b8ca123456789 89.160.20.156 89.160.20.156 49761 3389 6 20 4249 1418530010 1418530070 REJECT OK", + "kind": "event", + "start": "2014-12-14T04:06:50.000Z", + "end": "2014-12-14T04:07:50.000Z", + "type": "flow", + "category": "network_traffic", + "outcome": "deny" + }, + "aws": { + "vpcflow": { + "action": "REJECT", + "account_id": "123456789010", + "log_status": "OK", + "interface_id": "eni-1235b8ca123456789", + "version": "2" + } + } + }, + { + "destination": { + "port": 0, + "address": "172.31.16.139", + "ip": "172.31.16.139" + }, + "source": { + "geo": { + "continent_name": "Europe", + "region_iso_code": "SE-AB", + "city_name": "Tumba", + "country_iso_code": "SE", + "country_name": "Sweden", + "region_name": "Stockholm", + "location": { + "lon": 17.8167, + "lat": 59.2 + } + }, + "as": { + "number": 29518, + "organization": { + "name": "Bredband2 AB" + } + }, + "address": "89.160.20.156", + "port": 0, + "bytes": 336, + "ip": "89.160.20.156", + "packets": 4 + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "community_id": "1:cttDCHp3UNR8SFNTOgVYpAceHf4=", + "type": "ipv4", + "bytes": 336, + "iana_number": "1", + "packets": 4 + }, + "cloud": { + "provider": "aws", + "account": { + "id": "123456789010" + } + }, + "@timestamp": "2015-05-29T16:32:22.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "ip": [ + "89.160.20.156", + "172.31.16.139" + ] + }, + "event": { + "ingested": "2021-12-09T16:12:00.503416200Z", + "original": "2 123456789010 eni-1235b8ca123456789 89.160.20.156 172.31.16.139 0 0 1 4 336 1432917027 1432917142 ACCEPT OK", + "kind": "event", + "start": "2015-05-29T16:30:27.000Z", + "end": "2015-05-29T16:32:22.000Z", + "type": "flow", + "category": "network_traffic", + "outcome": "allow" + }, + "aws": { + "vpcflow": { + "action": "ACCEPT", + "account_id": "123456789010", + "log_status": "OK", + "interface_id": "eni-1235b8ca123456789", + "version": "2" + } + } + }, + { + "destination": { + "geo": { + "continent_name": "Europe", + "region_iso_code": "SE-AB", + "city_name": "Tumba", + "country_iso_code": "SE", + "country_name": "Sweden", + "region_name": "Stockholm", + "location": { + "lon": 17.8167, + "lat": 59.2 + } + }, + "as": { + "number": 29518, + "organization": { + "name": "Bredband2 AB" + } + }, + "address": "89.160.20.156", + "port": 0, + "ip": "89.160.20.156" + }, + "source": { + "address": "172.31.16.139", + "port": 0, + "bytes": 336, + "packets": 4, + "ip": "172.31.16.139" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "community_id": "1:XiVZKra6oEtIAPBi9QgeQL4Hp6M=", + "type": "ipv4", + "bytes": 336, + "iana_number": "1", + "packets": 4 + }, + "cloud": { + "provider": "aws", + "account": { + "id": "123456789010" + } + }, + "@timestamp": "2015-05-29T16:32:22.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "ip": [ + "172.31.16.139", + "89.160.20.156" + ] + }, + "event": { + "ingested": "2021-12-09T16:12:00.503420100Z", + "original": "2 123456789010 eni-1235b8ca123456789 172.31.16.139 89.160.20.156 0 0 1 4 336 1432917094 1432917142 REJECT OK", + "kind": "event", + "start": "2015-05-29T16:31:34.000Z", + "end": "2015-05-29T16:32:22.000Z", + "type": "flow", + "category": "network_traffic", + "outcome": "deny" + }, + "aws": { + "vpcflow": { + "action": "REJECT", + "account_id": "123456789010", + "log_status": "OK", + "interface_id": "eni-1235b8ca123456789", + "version": "2" + } + } + } + ] +} \ No newline at end of file diff --git a/test/packages/aws/data_stream/vpcflow/_dev/test/pipeline/test-tcp-flag-sequence.log b/test/packages/aws/data_stream/vpcflow/_dev/test/pipeline/test-tcp-flag-sequence.log index 4a05514cae..94b874fa6a 100644 --- a/test/packages/aws/data_stream/vpcflow/_dev/test/pipeline/test-tcp-flag-sequence.log +++ b/test/packages/aws/data_stream/vpcflow/_dev/test/pipeline/test-tcp-flag-sequence.log @@ -1,6 +1,3 @@ -version vpc-id subnet-id instance-id interface-id account-id type srcaddr dstaddr srcport dstport pkt-srcaddr pkt-dstaddr protocol bytes packets start end action tcp-flags log-status -3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 52.213.180.42 10.0.0.62 43416 5001 52.213.180.42 10.0.0.62 6 568 8 1566848875 1566848933 ACCEPT 2 OK -version vpc-id subnet-id instance-id interface-id account-id type srcaddr dstaddr srcport dstport pkt-srcaddr pkt-dstaddr protocol bytes packets start end action tcp-flags log-status +3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 89.160.20.156 10.0.0.62 43416 5001 89.160.20.156 10.0.0.62 6 568 8 1566848875 1566848933 ACCEPT 2 OK 3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 - - - - - - - - - - 1566848875 1566848933 - - SKIPDATA -version vpc-id subnet-id instance-id interface-id account-id type srcaddr dstaddr srcport dstport pkt-srcaddr pkt-dstaddr protocol bytes packets start end action tcp-flags log-status -3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 - - - - - - - - - - 1566848875 1566848933 - - NODATA +3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 - - - - - - - - - - 1566848875 1566848933 - - NODATA \ No newline at end of file diff --git a/test/packages/aws/data_stream/vpcflow/_dev/test/pipeline/test-tcp-flag-sequence.log-expected.json b/test/packages/aws/data_stream/vpcflow/_dev/test/pipeline/test-tcp-flag-sequence.log-expected.json index 9b89812b6e..cca3b2323c 100644 --- a/test/packages/aws/data_stream/vpcflow/_dev/test/pipeline/test-tcp-flag-sequence.log-expected.json +++ b/test/packages/aws/data_stream/vpcflow/_dev/test/pipeline/test-tcp-flag-sequence.log-expected.json @@ -1,58 +1,171 @@ { "expected": [ { - "cloud": { - "provider": "aws" + "destination": { + "port": 5001, + "address": "10.0.0.62", + "ip": "10.0.0.62" }, - "event": { - "kind": "event", - "original": "version vpc-id subnet-id instance-id interface-id account-id type srcaddr dstaddr srcport dstport pkt-srcaddr pkt-dstaddr protocol bytes packets start end action tcp-flags log-status" - } - }, - { - "cloud": { - "provider": "aws" + "source": { + "geo": { + "continent_name": "Europe", + "region_iso_code": "SE-AB", + "city_name": "Tumba", + "country_iso_code": "SE", + "country_name": "Sweden", + "region_name": "Stockholm", + "location": { + "lon": 17.8167, + "lat": 59.2 + } + }, + "as": { + "number": 29518, + "organization": { + "name": "Bredband2 AB" + } + }, + "address": "89.160.20.156", + "port": 43416, + "bytes": 568, + "ip": "89.160.20.156", + "packets": 8 }, - "event": { - "kind": "event", - "original": "3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 52.213.180.42 10.0.0.62 43416 5001 52.213.180.42 10.0.0.62 6 568 8 1566848875 1566848933 ACCEPT 2 OK" - } - }, - { - "cloud": { - "provider": "aws" + "tags": [ + "preserve_original_event" + ], + "network": { + "community_id": "1:dF5WY79X1yVncj+yH8q27Q5Bnpk=", + "transport": "tcp", + "type": "ipv4", + "bytes": 568, + "iana_number": "6", + "packets": 8 }, - "event": { - "kind": "event", - "original": "version vpc-id subnet-id instance-id interface-id account-id type srcaddr dstaddr srcport dstport pkt-srcaddr pkt-dstaddr protocol bytes packets start end action tcp-flags log-status" - } - }, - { "cloud": { - "provider": "aws" + "provider": "aws", + "account": { + "id": "123456789010" + }, + "instance": { + "id": "i-01234567890123456" + } + }, + "@timestamp": "2019-08-26T19:48:53.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "ip": [ + "89.160.20.156", + "10.0.0.62" + ] }, "event": { + "ingested": "2021-12-09T16:12:01.346119700Z", + "original": "3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 IPv4 89.160.20.156 10.0.0.62 43416 5001 89.160.20.156 10.0.0.62 6 568 8 1566848875 1566848933 ACCEPT 2 OK", "kind": "event", - "original": "3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 - - - - - - - - - - 1566848875 1566848933 - - SKIPDATA" + "start": "2019-08-26T19:47:55.000Z", + "end": "2019-08-26T19:48:53.000Z", + "type": "flow", + "category": "network_traffic", + "outcome": "allow" + }, + "aws": { + "vpcflow": { + "tcp_flags_array": [ + "syn" + ], + "vpc_id": "vpc-abcdefab012345678", + "pkt_srcaddr": "89.160.20.156", + "type": "IPv4", + "version": "3", + "instance_id": "i-01234567890123456", + "account_id": "123456789010", + "log_status": "OK", + "interface_id": "eni-1235b8ca123456789", + "tcp_flags": "2", + "subnet_id": "subnet-aaaaaaaa012345678", + "action": "ACCEPT", + "pkt_dstaddr": "10.0.0.62" + } } }, { "cloud": { - "provider": "aws" + "provider": "aws", + "account": { + "id": "123456789010" + }, + "instance": { + "id": "i-01234567890123456" + } + }, + "@timestamp": "2019-08-26T19:48:53.000Z", + "ecs": { + "version": "1.12.0" }, "event": { + "ingested": "2021-12-09T16:12:01.346125500Z", + "original": "3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 - - - - - - - - - - 1566848875 1566848933 - - SKIPDATA", "kind": "event", - "original": "version vpc-id subnet-id instance-id interface-id account-id type srcaddr dstaddr srcport dstport pkt-srcaddr pkt-dstaddr protocol bytes packets start end action tcp-flags log-status" - } + "start": "2019-08-26T19:47:55.000Z", + "end": "2019-08-26T19:48:53.000Z", + "type": "flow", + "category": "network_traffic" + }, + "aws": { + "vpcflow": { + "instance_id": "i-01234567890123456", + "account_id": "123456789010", + "log_status": "SKIPDATA", + "interface_id": "eni-1235b8ca123456789", + "vpc_id": "vpc-abcdefab012345678", + "subnet_id": "subnet-aaaaaaaa012345678", + "version": "3" + } + }, + "tags": [ + "preserve_original_event" + ] }, { "cloud": { - "provider": "aws" + "provider": "aws", + "account": { + "id": "123456789010" + }, + "instance": { + "id": "i-01234567890123456" + } + }, + "@timestamp": "2019-08-26T19:48:53.000Z", + "ecs": { + "version": "1.12.0" }, "event": { + "ingested": "2021-12-09T16:12:01.346129200Z", + "original": "3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 - - - - - - - - - - 1566848875 1566848933 - - NODATA", "kind": "event", - "original": "3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 - - - - - - - - - - 1566848875 1566848933 - - NODATA" - } + "start": "2019-08-26T19:47:55.000Z", + "end": "2019-08-26T19:48:53.000Z", + "type": "flow", + "category": "network_traffic" + }, + "aws": { + "vpcflow": { + "instance_id": "i-01234567890123456", + "account_id": "123456789010", + "log_status": "NODATA", + "interface_id": "eni-1235b8ca123456789", + "vpc_id": "vpc-abcdefab012345678", + "subnet_id": "subnet-aaaaaaaa012345678", + "version": "3" + } + }, + "tags": [ + "preserve_original_event" + ] } ] } \ No newline at end of file diff --git a/test/packages/aws/data_stream/vpcflow/agent/stream/aws-s3.yml.hbs b/test/packages/aws/data_stream/vpcflow/agent/stream/aws-s3.yml.hbs new file mode 100644 index 0000000000..ccf43bcddc --- /dev/null +++ b/test/packages/aws/data_stream/vpcflow/agent/stream/aws-s3.yml.hbs @@ -0,0 +1,51 @@ +queue_url: {{queue_url}} +{{#if credential_profile_name}} +credential_profile_name: {{credential_profile_name}} +{{/if}} +{{#if shared_credential_file}} +shared_credential_file: {{shared_credential_file}} +{{/if}} +{{#if visibility_timeout}} +visibility_timeout: {{visibility_timeout}} +{{/if}} +{{#if api_timeout}} +api_timeout: {{api_timeout}} +{{/if}} +{{#if max_number_of_messages}} +max_number_of_messages: {{max_number_of_messages}} +{{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} +{{#if access_key_id}} +access_key_id: {{access_key_id}} +{{/if}} +{{#if secret_access_key}} +secret_access_key: {{secret_access_key}} +{{/if}} +{{#if session_token}} +session_token: {{session_token}} +{{/if}} +{{#if role_arn}} +role_arn: {{role_arn}} +{{/if}} +{{#if fips_enabled}} +fips_enabled: {{fips_enabled}} +{{/if}} +{{#if proxy_url }} +proxy_url: {{proxy_url}} +{{/if}} +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} \ No newline at end of file diff --git a/test/packages/aws/data_stream/vpcflow/agent/stream/s3.yml.hbs b/test/packages/aws/data_stream/vpcflow/agent/stream/s3.yml.hbs deleted file mode 100644 index 8241ac6fcd..0000000000 --- a/test/packages/aws/data_stream/vpcflow/agent/stream/s3.yml.hbs +++ /dev/null @@ -1,139 +0,0 @@ -queue_url: {{queue_url}} -{{#if credential_profile_name}} -credential_profile_name: {{credential_profile_name}} -{{/if}} -{{#if shared_credential_file}} -shared_credential_file: {{shared_credential_file}} -{{/if}} -{{#if visibility_timeout}} -visibility_timeout: {{visibility_timeout}} -{{/if}} -{{#if api_timeout}} -api_timeout: {{api_timeout}} -{{/if}} -{{#if endpoint}} -endpoint: {{endpoint}} -{{/if}} -{{#if access_key_id}} -access_key_id: {{access_key_id}} -{{/if}} -{{#if secret_access_key}} -secret_access_key: {{secret_access_key}} -{{/if}} -{{#if session_token}} -session_token: {{session_token}} -{{/if}} -{{#if role_arn}} -role_arn: {{role_arn}} -{{/if}} -{{#if fips_enabled}} -fips_enabled: {{fips_enabled}} -{{/if}} -processors: - - drop_event: - when.regexp.message: "^version" - - drop_event: - when.regexp.message: "^instance-id" - - script: - lang: javascript - source: > - function process(event) { - var message = event.Get("message"); - var tokens = message.split(" ").length; - event.Put("@metadata.message_token_count", tokens); - } - # Default vpc flow log format - - dissect: - when: - equals: - '@metadata.message_token_count': 14 - field: message - target_prefix: aws.vpcflow - tokenizer: '%{version} %{account_id} %{interface_id} %{srcaddr} %{dstaddr} %{srcport} %{dstport} %{protocol} %{packets} %{bytes} %{start} %{end} %{action} %{log_status}' - # Custom flow log for traffic through a NAT gateway - - dissect: - when: - equals: - '@metadata.message_token_count': 6 - field: message - target_prefix: aws.vpcflow - tokenizer: '%{instance_id} %{interface_id} %{srcaddr} %{dstaddr} %{pkt_srcaddr} %{pkt_dstaddr}' - # Custom flow log for traffic through a transit gateway - - dissect: - when: - equals: - '@metadata.message_token_count': 17 - field: message - target_prefix: aws.vpcflow - tokenizer: '%{version} %{interface_id} %{account_id} %{vpc_id} %{subnet_id} %{instance_id} %{srcaddr} %{dstaddr} %{srcport} %{dstport} %{protocol} %{tcp_flags} %{type} %{pkt_srcaddr} %{pkt_dstaddr} %{action} %{log_status}' - # TCP Flag Sequence - - dissect: - when: - equals: - '@metadata.message_token_count': 21 - field: message - target_prefix: aws.vpcflow - tokenizer: '%{version} %{vpc_id} %{subnet_id} %{instance_id} %{interface_id} %{account_id} %{type} %{srcaddr} %{dstaddr} %{srcport} %{dstport} %{pkt_srcaddr} %{pkt_dstaddr} %{protocol} %{bytes} %{packets} %{start} %{end} %{action} %{tcp_flags} %{log_status}' - - convert: - ignore_missing: true - fields: - - {from: aws.vpcflow.srcaddr, to: source.address} - - {from: aws.vpcflow.srcaddr, to: source.ip, type: ip} - - {from: aws.vpcflow.srcport, to: source.port, type: long} - - {from: aws.vpcflow.dstaddr, to: destination.address} - - {from: aws.vpcflow.dstaddr, to: destination.ip, type: ip} - - {from: aws.vpcflow.dstport, to: destination.port, type: long} - - {from: aws.vpcflow.protocol, to: network.iana_number, type: string} - - {from: aws.vpcflow.packets, to: source.packets, type: long} - - {from: aws.vpcflow.bytes, to: source.bytes, type: long} - - {from: aws.vpcflow.packets, to: network.packets, type: long} - - {from: aws.vpcflow.bytes, to: network.bytes, type: long} - - drop_fields: - fields: ["aws.vpcflow.srcaddr", "aws.vpcflow.srcport", "aws.vpcflow.dstaddr", "aws.vpcflow.dstport", "aws.vpcflow.bytes", "aws.vpcflow.packets", "aws.vpcflow.protocol"] - - community_id: ~ - # Use the aws.vpcflow.action value to set the event.outcome value to either "allow" or "deny". - - add_fields: - when.equals.aws.vpcflow.action: ACCEPT - target: event - fields: {outcome: allow} - - add_fields: - when.equals.aws.vpcflow.action: REJECT - target: event - fields: {outcome: deny} - - add_fields: - target: event - fields: {type: flow} - - add_fields: - target: event - fields: {category: network_traffic} - # Add network.type: ipv4 or ipv6 - - if: - contains.source.ip: "." - then: - - add_fields: - target: network - fields: {type: ipv4} - - if: - contains.source.ip: ":" - then: - - add_fields: - target: network - fields: {type: ipv6} - # Add network.transport: based on IANA protocol number of the traffic - # http://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml - - if: - equals.network.iana_number: "6" - then: - - add_fields: - target: network - fields: {transport: tcp} - - if: - equals.network.iana_number: "17" - then: - - add_fields: - target: network - fields: {transport: udp} - - add_fields: - target: '' - fields: - ecs.version: 1.5.0 \ No newline at end of file diff --git a/test/packages/aws/data_stream/vpcflow/elasticsearch/ingest_pipeline/default.yml b/test/packages/aws/data_stream/vpcflow/elasticsearch/ingest_pipeline/default.yml index 811ec69692..d4d98f0839 100644 --- a/test/packages/aws/data_stream/vpcflow/elasticsearch/ingest_pipeline/default.yml +++ b/test/packages/aws/data_stream/vpcflow/elasticsearch/ingest_pipeline/default.yml @@ -2,33 +2,75 @@ description: Pipeline for AWS VPC Flow Logs processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + - set: + field: ecs.version + value: '1.12.0' + - rename: + field: message + target_field: event.original + ignore_missing: true + - set: + field: event.type + value: flow + - set: + field: event.category + value: network_traffic + - drop: + if: 'ctx.event?.original.startsWith("version") || ctx.event?.original.startsWith("instance-id")' + - script: + lang: painless + if: ctx.event?.original != null + source: >- + ctx._temp_ = new HashMap(); + ctx._temp_.message_token_count = ctx.event?.original.splitOnToken(" ").length; + - dissect: + field: event.original + pattern: '%{aws.vpcflow.version} %{aws.vpcflow.account_id} %{aws.vpcflow.interface_id} %{aws.vpcflow.srcaddr} %{aws.vpcflow.dstaddr} %{aws.vpcflow.srcport} %{aws.vpcflow.dstport} %{aws.vpcflow.protocol} %{aws.vpcflow.packets} %{aws.vpcflow.bytes} %{aws.vpcflow.start} %{aws.vpcflow.end} %{aws.vpcflow.action} %{aws.vpcflow.log_status}' + if: ctx?._temp_?.message_token_count == 14 + - dissect: + field: event.original + pattern: '%{aws.vpcflow.instance_id} %{aws.vpcflow.interface_id} %{aws.vpcflow.srcaddr} %{aws.vpcflow.dstaddr} %{aws.vpcflow.pkt_srcaddr} %{aws.vpcflow.pkt_dstaddr}' + if: ctx?._temp_?.message_token_count == 6 + - dissect: + field: event.original + pattern: '%{aws.vpcflow.version} %{aws.vpcflow.interface_id} %{aws.vpcflow.account_id} %{aws.vpcflow.vpc_id} %{aws.vpcflow.subnet_id} %{aws.vpcflow.instance_id} %{aws.vpcflow.srcaddr} %{aws.vpcflow.dstaddr} %{aws.vpcflow.srcport} %{aws.vpcflow.dstport} %{aws.vpcflow.protocol} %{aws.vpcflow.tcp_flags} %{aws.vpcflow.type} %{aws.vpcflow.pkt_srcaddr} %{aws.vpcflow.pkt_dstaddr} %{aws.vpcflow.action} %{aws.vpcflow.log_status}' + if: ctx?._temp_?.message_token_count == 17 + - dissect: + field: event.original + pattern: '%{aws.vpcflow.version} %{aws.vpcflow.vpc_id} %{aws.vpcflow.subnet_id} %{aws.vpcflow.instance_id} %{aws.vpcflow.interface_id} %{aws.vpcflow.account_id} %{aws.vpcflow.type} %{aws.vpcflow.srcaddr} %{aws.vpcflow.dstaddr} %{aws.vpcflow.srcport} %{aws.vpcflow.dstport} %{aws.vpcflow.pkt_srcaddr} %{aws.vpcflow.pkt_dstaddr} %{aws.vpcflow.protocol} %{aws.vpcflow.bytes} %{aws.vpcflow.packets} %{aws.vpcflow.start} %{aws.vpcflow.end} %{aws.vpcflow.action} %{aws.vpcflow.tcp_flags} %{aws.vpcflow.log_status}' + if: ctx?._temp_?.message_token_count == 21 + # Convert Unix epoch to timestamp - date: - field: "aws.vpcflow.end" - target_field: "@timestamp" + field: aws.vpcflow.end + target_field: '@timestamp' ignore_failure: true formats: - UNIX - date: - field: "aws.vpcflow.start" - target_field: "event.start" + field: aws.vpcflow.start + target_field: event.start ignore_failure: true formats: - UNIX - date: - field: "aws.vpcflow.end" - target_field: "event.end" + field: aws.vpcflow.end + target_field: event.end ignore_failure: true formats: - UNIX - remove: - field: ["aws.vpcflow.start", "aws.vpcflow.end"] + field: + - aws.vpcflow.start + - aws.vpcflow.end ignore_missing: true - - script: lang: painless ignore_failure: true - if: ctx?.aws != null + if: ctx.aws != null source: >- void handleMap(Map map) { for (def x : map.values()) { @@ -50,7 +92,81 @@ processors: } } handleMap(ctx.aws); - + - set: + field: event.outcome + value: allow + if: ctx.aws?.vpcflow?.action == "ACCEPT" + - set: + field: event.outcome + value: deny + if: ctx.aws?.vpcflow?.action == "REJECT" + - rename: + field: aws.vpcflow.srcaddr + target_field: source.address + ignore_missing: true + - set: + field: source.ip + copy_from: source.address + if: ctx.source?.address != null + - convert: + field: aws.vpcflow.srcport + target_field: source.port + type: integer + ignore_missing: true + - rename: + field: aws.vpcflow.dstaddr + target_field: destination.address + ignore_missing: true + - set: + field: destination.ip + copy_from: destination.address + if: ctx.destination?.address != null + - convert: + field: aws.vpcflow.dstport + target_field: destination.port + type: integer + ignore_missing: true + - rename: + field: aws.vpcflow.protocol + target_field: network.iana_number + ignore_missing: true + - convert: + field: aws.vpcflow.packets + target_field: source.packets + type: long + ignore_missing: true + - convert: + field: aws.vpcflow.bytes + target_field: source.bytes + type: long + ignore_missing: true + - set: + field: network.bytes + copy_from: source.bytes + if: ctx.source?.bytes != null + - set: + field: network.packets + copy_from: source.packets + if: ctx.source?.packets != null + - set: + field: network.type + value: ipv4 + if: 'ctx.source?.ip != null && ctx.source?.ip.contains(".")' + - set: + field: network.type + value: ipv6 + if: 'ctx.source?.ip != null && ctx.source?.ip.contains(":")' + - set: + field: network.transport + value: tcp + if: ctx.network?.iana_number == "6" + - set: + field: network.transport + value: udp + if: ctx.network?.iana_number == "17" + - community_id: + target_field: network.community_id + ignore_failure: true # IP Geolocation Lookup - geoip: field: source.ip @@ -60,7 +176,6 @@ processors: field: destination.ip target_field: destination.geo ignore_missing: true - # IP Autonomous System (AS) Lookup - geoip: database_file: GeoLite2-ASN.mmdb @@ -78,7 +193,6 @@ processors: - asn - organization_name ignore_missing: true - - rename: field: source.as.asn target_field: source.as.number @@ -95,36 +209,72 @@ processors: field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - - rename: - field: message - target_field: event.original - ignore_missing: true - # Generate related.ip field - append: - if: ctx.source?.ip != null && ctx.destination?.ip != null + if: 'ctx.source?.ip != null && ctx.destination?.ip != null' field: related.ip value: ["{{source.ip}}", "{{destination.ip}}"] - - set: field: cloud.provider value: aws - - set: - if: "ctx?.aws?.vpcflow?.account_id != null" + if: ctx.aws?.vpcflow?.account_id != null field: cloud.account.id - value: "{{aws.vpcflow.account_id}}" - + value: '{{aws.vpcflow.account_id}}' - set: - if: "ctx?.aws?.vpcflow?.instance_id != null && ctx.aws.vpcflow.instance_id != '-'" + if: 'ctx?.aws?.vpcflow?.instance_id != null && ctx.aws.vpcflow.instance_id != "-"' field: cloud.instance.id - value: "{{aws.vpcflow.instance_id}}" - + value: '{{aws.vpcflow.instance_id}}' - set: field: event.kind value: event + - script: + lang: painless + ignore_failure: true + if: "ctx.aws?.vpcflow?.tcp_flags != null" + source: | + if (ctx.aws.vpcflow.tcp_flags_array == null) { + ArrayList al = new ArrayList(); + ctx.aws.vpcflow.put("tcp_flags_array", al); + } + def flags = Integer.parseUnsignedInt(ctx.aws.vpcflow.tcp_flags); + + if ((flags & 0x01) != 0) { + ctx.aws.vpcflow.tcp_flags_array.add('fin'); + } + if ((flags & 0x02) != 0) { + ctx.aws.vpcflow.tcp_flags_array.add('syn'); + } + if ((flags & 0x04) != 0) { + ctx.aws.vpcflow.tcp_flags_array.add('rst'); + } + if ((flags & 0x08) != 0) { + ctx.aws.vpcflow.tcp_flags_array.add('psh'); + } + if ((flags & 0x10) != 0) { + ctx.aws.vpcflow.tcp_flags_array.add('ack'); + } + if ((flags & 0x20) != 0) { + ctx.aws.vpcflow.tcp_flags_array.add('urg'); + } + - remove: + field: + - _temp_ + - aws.vpcflow.srcaddr + - aws.vpcflow.srcport + - aws.vpcflow.dstaddr + - aws.vpcflow.dstport + - aws.vpcflow.bytes + - aws.vpcflow.packets + - aws.vpcflow.protocol + ignore_missing: true + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - set: - field: "error.message" - value: "{{ _ingest.on_failure_message }}" + field: 'error.message' + value: '{{ _ingest.on_failure_message }}' diff --git a/test/packages/aws/data_stream/vpcflow/fields/base-fields.yml b/test/packages/aws/data_stream/vpcflow/fields/base-fields.yml index 7c798f4534..8360b70d7e 100644 --- a/test/packages/aws/data_stream/vpcflow/fields/base-fields.yml +++ b/test/packages/aws/data_stream/vpcflow/fields/base-fields.yml @@ -10,3 +10,11 @@ - name: '@timestamp' type: date description: Event timestamp. +- name: event.module + type: constant_keyword + description: Event module + value: aws +- name: event.dataset + type: constant_keyword + description: Event dataset + value: aws.vpcflow diff --git a/test/packages/aws/data_stream/vpcflow/fields/ecs.yml b/test/packages/aws/data_stream/vpcflow/fields/ecs.yml new file mode 100644 index 0000000000..6fbd292a09 --- /dev/null +++ b/test/packages/aws/data_stream/vpcflow/fields/ecs.yml @@ -0,0 +1,94 @@ +- name: cloud.account.id + external: ecs +- name: cloud.instance.id + external: ecs +- name: cloud.provider + external: ecs +- name: destination.address + external: ecs +- name: destination.as.number + external: ecs +- name: destination.as.organization.name + external: ecs +- name: destination.geo.city_name + external: ecs +- name: destination.geo.continent_name + external: ecs +- name: destination.geo.country_iso_code + external: ecs +- name: destination.geo.country_name + external: ecs +- name: destination.geo.location + external: ecs +- name: destination.geo.region_iso_code + external: ecs +- name: destination.geo.region_name + external: ecs +- name: destination.ip + external: ecs +- name: destination.port + external: ecs +- name: ecs.version + external: ecs +- name: error.message + external: ecs +- name: event.category + external: ecs +- name: event.end + external: ecs +- name: event.kind + external: ecs +- name: event.original + external: ecs +- name: event.outcome + external: ecs +- name: event.start + external: ecs +- name: event.type + external: ecs +- name: network.bytes + external: ecs +- name: network.community_id + external: ecs +- name: network.iana_number + external: ecs +- name: network.packets + external: ecs +- name: network.transport + external: ecs +- name: network.type + external: ecs +- name: related.ip + external: ecs +- name: source.address + external: ecs +- name: source.as.number + external: ecs +- name: source.as.organization.name + external: ecs +- name: source.as.organization.name + external: ecs +- name: source.bytes + external: ecs +- name: source.geo.city_name + external: ecs +- name: source.geo.continent_name + external: ecs +- name: source.geo.country_iso_code + external: ecs +- name: source.geo.country_name + external: ecs +- name: source.geo.location + external: ecs +- name: source.geo.region_iso_code + external: ecs +- name: source.geo.region_name + external: ecs +- name: source.ip + external: ecs +- name: source.packets + external: ecs +- name: source.port + external: ecs +- name: tags + external: ecs diff --git a/test/packages/aws/data_stream/vpcflow/fields/fields.yml b/test/packages/aws/data_stream/vpcflow/fields/fields.yml index 0ac3cfb533..eb86456985 100644 --- a/test/packages/aws/data_stream/vpcflow/fields/fields.yml +++ b/test/packages/aws/data_stream/vpcflow/fields/fields.yml @@ -45,130 +45,12 @@ type: keyword description: | The bitmask value for the following TCP flags: 2=SYN,18=SYN-ACK,1=FIN,4=RST + - name: tcp_flags_array + type: keyword + description: > + List of TCP flags: 'fin, syn, rst, psh, ack, urg' + - name: type type: keyword description: | The type of traffic: IPv4, IPv6, or EFA. -- name: event.start - type: date - description: event.start contains the date when the event started or when the activity was first observed. -- name: event.end - type: date - description: event.end contains the date when the event ended or when the activity was last observed. -- name: destination.geo.continent_name - type: keyword - description: Name of the continent. -- name: destination.geo.country_iso_code - type: keyword - description: Country ISO code. -- name: destination.geo.location - type: geo_point - description: Longitude and latitude. -- name: destination.ip - type: ip - description: IP address of the destination. -- name: destination.address - type: keyword - description: Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the .address field. -- name: destination.port - type: long - description: Port of the destination. -- name: event.category - type: keyword - description: Event category (e.g. database) -- name: event.outcome - type: keyword - description: This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. -- name: event.type - type: keyword - description: Event severity (e.g. info, error) -- name: source.as.number - type: long - description: Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. -- name: source.as.organization.name - type: keyword - description: Organization name. -- name: destination.as.number - type: long - description: Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. -- name: destination.as.organization.name - type: keyword - description: Organization name. -- name: event.original - type: keyword - description: Raw text message of entire event. Used to demonstrate log integrity. -- name: cloud.account.id - type: keyword - description: The cloud account or organization id used to identify different entities in a multi-tenant environment. -- name: cloud.instance.id - type: keyword - description: Instance ID of the host machine. -- name: cloud.provider - type: keyword - description: Name of the cloud provider. -- name: related.ip - type: ip - description: All of the IPs seen on your event. -- name: event.kind - type: keyword - description: Event kind (e.g. event, alert, metric, state, pipeline_error, signal) -- name: cloud.account.id - type: keyword - description: The cloud account or organization id used to identify different entities in a multi-tenant environment. -- name: network.bytes - type: long - description: Total bytes transferred in both directions. -- name: network.community_id - type: keyword - description: A hash of source and destination IPs and ports, as well as the protocol used in a communication. This is a tool-agnostic standard to identify flows. -- name: network.iana_number - type: keyword - description: IANA Protocol Number (https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml). Standardized list of protocols. This aligns well with NetFlow and sFlow related logs which use the IANA Protocol Number. -- name: network.packets - type: long - description: Total packets transferred in both directions. -- name: network.transport - type: keyword - description: Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) -- name: network.type - type: keyword - description: In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc -- name: source.address - type: keyword - description: Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the .address field. -- name: source.as.number - type: long - description: Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. -- name: source.as.organization.name - type: keyword - description: Organization name. -- name: source.bytes - type: long - description: Bytes sent from the source to the destination. -- name: source.geo.city_name - type: keyword - description: City name. -- name: source.geo.continent_name - type: keyword - description: Name of the continent. -- name: source.geo.country_iso_code - type: keyword - description: Country ISO code. -- name: source.geo.location - type: geo_point - description: Longitude and latitude. -- name: source.geo.region_iso_code - type: keyword - description: Region ISO code. -- name: source.geo.region_name - type: keyword - description: Region name. -- name: source.ip - type: ip - description: IP address of the source (IPv4 or IPv6). -- name: source.packets - type: long - description: Packets sent from the source to the destination. -- name: source.port - type: long - description: Port of the source. diff --git a/test/packages/aws/data_stream/vpcflow/manifest.yml b/test/packages/aws/data_stream/vpcflow/manifest.yml index a27c351a81..3812639fb8 100644 --- a/test/packages/aws/data_stream/vpcflow/manifest.yml +++ b/test/packages/aws/data_stream/vpcflow/manifest.yml @@ -1,12 +1,25 @@ title: AWS vpcflow logs -release: beta type: logs streams: - - input: s3 - template_path: s3.yml.hbs + - input: aws-s3 + template_path: aws-s3.yml.hbs title: AWS vpcflow logs description: Collect AWS vpcflow logs using s3 input vars: + - name: visibility_timeout + type: text + title: Visibility Timeout + multi: false + required: false + show_user: false + description: The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours. + - name: api_timeout + type: text + title: API Timeout + multi: false + required: false + show_user: false + description: The maximum duration of AWS API can take. The maximum is half of the visibility timeout value. - name: queue_url type: text title: Queue URL @@ -22,3 +35,36 @@ streams: required: false show_user: false description: Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - aws-vpcflow + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: max_number_of_messages + type: integer + title: Maximum Concurrent SQS Messages + description: The maximum number of SQS messages that can be inflight at any time. + default: 5 + required: false + show_user: false diff --git a/test/packages/aws/data_stream/vpcflow/sample_event.json b/test/packages/aws/data_stream/vpcflow/sample_event.json new file mode 100644 index 0000000000..b84f7671cf --- /dev/null +++ b/test/packages/aws/data_stream/vpcflow/sample_event.json @@ -0,0 +1,65 @@ +{ + "data_stream": { + "namespace": "default", + "type": "logs", + "dataset": "aws.vpcflow" + }, + "destination": { + "port": 22, + "address": "2001:db8:1234:a102:3304:8879:34cf:4071", + "ip": "2001:db8:1234:a102:3304:8879:34cf:4071" + }, + "source": { + "address": "2001:db8:1234:a100:8d6e:3477:df66:f105", + "port": 34892, + "bytes": 8855, + "packets": 54, + "ip": "2001:db8:1234:a100:8d6e:3477:df66:f105" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "community_id": "1:hXZclvxUJScaVf0xMIJR6yW6tBQ=", + "transport": "tcp", + "type": "ipv6", + "bytes": 8855, + "iana_number": "6", + "packets": 54 + }, + "cloud": { + "provider": "aws", + "account": { + "id": "123456789010" + } + }, + "@timestamp": "2016-10-31T11:37:00.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "ip": [ + "2001:db8:1234:a100:8d6e:3477:df66:f105", + "2001:db8:1234:a102:3304:8879:34cf:4071" + ] + }, + "event": { + "ingested": "2021-09-28T19:10:43.075027100Z", + "original": "2 123456789010 eni-1235b8ca123456789 2001:db8:1234:a100:8d6e:3477:df66:f105 2001:db8:1234:a102:3304:8879:34cf:4071 34892 22 6 54 8855 1477913708 1477913820 ACCEPT OK", + "kind": "event", + "start": "2016-10-31T11:35:08.000Z", + "end": "2016-10-31T11:37:00.000Z", + "type": "flow", + "category": "network_traffic", + "outcome": "allow" + }, + "aws": { + "vpcflow": { + "action": "ACCEPT", + "account_id": "123456789010", + "log_status": "OK", + "interface_id": "eni-1235b8ca123456789", + "version": "2" + } + } +} \ No newline at end of file diff --git a/test/packages/aws/data_stream/vpn/agent/stream/stream.yml.hbs b/test/packages/aws/data_stream/vpn/agent/stream/stream.yml.hbs index a22a1d98e0..49496da3ae 100644 --- a/test/packages/aws/data_stream/vpn/agent/stream/stream.yml.hbs +++ b/test/packages/aws/data_stream/vpn/agent/stream/stream.yml.hbs @@ -29,4 +29,7 @@ latency: {{latency}} {{/if}} {{#if tags_filter}} tags_filter: {{tags_filter}} +{{/if}} +{{#if proxy_url }} +proxy_url: {{proxy_url}} {{/if}} \ No newline at end of file diff --git a/test/packages/aws/data_stream/vpn/fields/base-fields.yml b/test/packages/aws/data_stream/vpn/fields/base-fields.yml index 7c798f4534..6e588484d6 100644 --- a/test/packages/aws/data_stream/vpn/fields/base-fields.yml +++ b/test/packages/aws/data_stream/vpn/fields/base-fields.yml @@ -10,3 +10,11 @@ - name: '@timestamp' type: date description: Event timestamp. +- name: event.module + type: constant_keyword + description: Event module + value: aws +- name: event.dataset + type: constant_keyword + description: Event dataset + value: aws.vpn diff --git a/test/packages/aws/data_stream/vpn/fields/ecs.yml b/test/packages/aws/data_stream/vpn/fields/ecs.yml index a02d7269c5..83e3f6f122 100644 --- a/test/packages/aws/data_stream/vpn/fields/ecs.yml +++ b/test/packages/aws/data_stream/vpn/fields/ecs.yml @@ -1,53 +1,24 @@ -- name: cloud - title: Cloud - group: 2 - type: group - footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' - fields: - - name: account.id - level: extended - type: keyword - description: |- - The cloud account or organization id used to identify different entities in a multi-tenant environment. - Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. - ignore_above: 1024 - - name: account.name - level: extended - type: keyword - description: |- - The cloud account name or alias used to identify different entities in a multi-tenant environment. - Examples: AWS account name, Google Cloud ORG display name. - ignore_above: 1024 - - name: availability_zone - level: extended - type: keyword - description: Availability zone in which this host is running. - ignore_above: 1024 - - name: instance.id - level: extended - type: keyword - description: Instance ID of the host machine. - ignore_above: 1024 - - name: machine.type - level: extended - type: keyword - description: Machine type of the host machine. - ignore_above: 1024 - - name: provider - level: extended - type: keyword - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. - ignore_above: 1024 - - name: region - level: extended - type: keyword - description: Region in which this host is running. - ignore_above: 1024 -- name: ecs.version - type: keyword - description: ECS version this event conforms to. - example: 1.0.0 - ignore_above: 1024 -- name: service.type - type: keyword - description: Service type +- external: ecs + name: cloud +- external: ecs + name: cloud.account.id +- external: ecs + name: cloud.account.name +- external: ecs + name: cloud.availability_zone +- external: ecs + name: cloud.instance.id +- external: ecs + name: cloud.machine.type +- external: ecs + name: cloud.provider +- external: ecs + name: cloud.region +- external: ecs + name: ecs.version +- external: ecs + name: error +- external: ecs + name: error.message +- external: ecs + name: service.type diff --git a/test/packages/aws/data_stream/vpn/manifest.yml b/test/packages/aws/data_stream/vpn/manifest.yml index 7daa957da1..aceda97431 100644 --- a/test/packages/aws/data_stream/vpn/manifest.yml +++ b/test/packages/aws/data_stream/vpn/manifest.yml @@ -1,5 +1,4 @@ title: AWS VPN metrics -release: beta type: metrics streams: - input: aws/metrics diff --git a/test/packages/aws/data_stream/waf/_dev/test/pipeline/test-common-config.yml b/test/packages/aws/data_stream/waf/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 0000000000..5622947e4b --- /dev/null +++ b/test/packages/aws/data_stream/waf/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,5 @@ +dynamic_fields: + event.ingested: ".*" +fields: + tags: + - preserve_original_event diff --git a/test/packages/aws/data_stream/waf/_dev/test/pipeline/test-waf.log b/test/packages/aws/data_stream/waf/_dev/test/pipeline/test-waf.log new file mode 100644 index 0000000000..774353168e --- /dev/null +++ b/test/packages/aws/data_stream/waf/_dev/test/pipeline/test-waf.log @@ -0,0 +1,4 @@ +{"timestamp":1576280412771,"formatVersion":1,"webaclId":"arn:aws:wafv2:ap-southeast-2:EXAMPLE12345:regional/webacl/STMTest/1EXAMPLE-2ARN-3ARN-4ARN-123456EXAMPLE","terminatingRuleId":"STMTest_SQLi_XSS","terminatingRuleType":"REGULAR","action":"BLOCK","terminatingRuleMatchDetails":[{"conditionType":"SQL_INJECTION","location":"HEADER","matchedData":["10","AND","1"]}],"httpSourceName":"-","httpSourceId":"-","ruleGroupList":[],"rateBasedRuleList":[],"nonTerminatingMatchingRules":[],"httpRequest":{"clientIp":"89.160.20.156","country":"AU","headers":[{"name":"Host","value":"localhost:1989"},{"name":"User-Agent","value":"curl/7.61.1"},{"name":"Accept","value":"*/*"},{"name":"x-stm-test","value":"10 AND 1=1"}],"uri":"/foo","args":"","httpVersion":"HTTP/1.1","httpMethod":"GET","requestId":"rid"},"labels":[{"name":"value"}]} +{"timestamp":1592357192516,"formatVersion":1,"webaclId":"arn:aws:wafv2:us-east-1:123456789012:global/webacl/hello-world/5933d6d9-9dde-js82-v8aw-9ck28nv9","terminatingRuleId":"Default_Action","terminatingRuleType":"REGULAR","action":"ALLOW","terminatingRuleMatchDetails":[],"httpSourceName":"-","httpSourceId":"-","ruleGroupList":[],"rateBasedRuleList":[],"nonTerminatingMatchingRules":[{"ruleId":"TestRule","action":"COUNT","ruleMatchDetails":[{"conditionType":"SQL_INJECTION","location":"HEADER","matchedData":["10","and","1"]}]}],"httpRequest":{"clientIp":"89.160.20.156","country":"US","headers":[{"name":"Host","value":"localhost:1989"},{"name":"User-Agent","value":"curl/7.61.1"},{"name":"Accept","value":"*/*"},{"name":"foo","value":"10 AND 1=1"}],"uri":"/foo","args":"","httpVersion":"HTTP/1.1","httpMethod":"GET","requestId":"rid"},"labels":[{"name":"value"}]} +{"timestamp":1592361810888,"formatVersion":1,"webaclId":"arn:aws:wafv2:us-east-1:123456789012:global/webacl/hello-world/5933d6d9-9dde-js82-v8aw-9ck28nv9","terminatingRuleId":"RG-Reference","terminatingRuleType":"GROUP","action":"BLOCK","terminatingRuleMatchDetails":[{"conditionType":"XSS","location":"HEADER","matchedData":["<","frameset"]}],"httpSourceName":"-","httpSourceId":"-","ruleGroupList":[{"ruleGroupId":"arn:aws:wafv2:us-east-1:123456789012:global/rulegroup/hello-world/c05lb698-1f11-4m41-aef4-99a506d53f4b","terminatingRule":{"ruleId":"RuleA-XSS","action":"BLOCK","ruleMatchDetails":null},"nonTerminatingMatchingRules":[{"ruleId":"RuleB-SQLi","action":"COUNT","ruleMatchDetails":[{"conditionType":"SQL_INJECTION","location":"HEADER","matchedData":["10","and","1"]}]}],"excludedRules":null}],"rateBasedRuleList":[],"nonTerminatingMatchingRules":[],"httpRequest":{"clientIp":"89.160.20.156","country":"US","headers":[{"name":"Host","value":"localhost:1989"},{"name":"User-Agent","value":"curl/7.61.1"},{"name":"Accept","value":"*/*"},{"name":"xssfoo","value":""},{"name":"bar","value":"10 AND 1=1"}],"uri":"/foo","args":"","httpVersion":"HTTP/1.1","httpMethod":"GET","requestId":"rid"},"labels":[{"name":"value"}]} +{"timestamp":1576280412771,"formatVersion":1,"webaclId":"arn:aws:wafv2:ap-southeast-2:12345:regional/webacl/test/111","terminatingRuleId":"STMTest_SQLi_XSS","terminatingRuleType":"REGULAR","action":"BLOCK","terminatingRuleMatchDetails":[{"conditionType":"SQL_INJECTION","location":"UNKNOWN","matchedData":["10","AND","1"]}],"httpSourceName":"ALB","httpSourceId":"alb","ruleGroupList":[],"rateBasedRuleList":[],"nonTerminatingMatchingRules":[],"requestHeadersInserted":null,"responseCodeSent":null,"httpRequest":{"clientIp":"89.160.20.156","country":"AU","headers":[],"uri":"","args":"","httpVersion":"HTTP/1.1","httpMethod":"POST","requestId":"null"},"labels":[{"name":"value"}]} \ No newline at end of file diff --git a/test/packages/aws/data_stream/waf/_dev/test/pipeline/test-waf.log-expected.json b/test/packages/aws/data_stream/waf/_dev/test/pipeline/test-waf.log-expected.json new file mode 100644 index 0000000000..b2240aa9ad --- /dev/null +++ b/test/packages/aws/data_stream/waf/_dev/test/pipeline/test-waf.log-expected.json @@ -0,0 +1,421 @@ +{ + "expected": [ + { + "rule": { + "ruleset": "REGULAR", + "id": "STMTest_SQLi_XSS" + }, + "source": { + "geo": { + "continent_name": "Europe", + "region_iso_code": "SE-AB", + "city_name": "Tumba", + "country_iso_code": "SE", + "country_name": "Sweden", + "region_name": "Stockholm", + "location": { + "lon": 17.8167, + "lat": 59.2 + } + }, + "as": { + "number": 29518, + "organization": { + "name": "Bredband2 AB" + } + }, + "ip": "89.160.20.156" + }, + "url": { + "path": "/foo" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "http", + "transport": "tcp" + }, + "cloud": { + "region": "ap-southeast-2", + "provider": "aws", + "service": { + "name": "wafv2" + }, + "account": { + "id": "EXAMPLE12345" + } + }, + "ecs": { + "version": "1.12.0" + }, + "related": { + "ip": [ + "89.160.20.156" + ] + }, + "http": { + "request": { + "method": "GET", + "id": "rid" + }, + "version": "1.1" + }, + "event": { + "action": "BLOCK", + "ingested": "2021-12-09T16:12:01.711621Z", + "original": "{\"timestamp\":1576280412771,\"formatVersion\":1,\"webaclId\":\"arn:aws:wafv2:ap-southeast-2:EXAMPLE12345:regional/webacl/STMTest/1EXAMPLE-2ARN-3ARN-4ARN-123456EXAMPLE\",\"terminatingRuleId\":\"STMTest_SQLi_XSS\",\"terminatingRuleType\":\"REGULAR\",\"action\":\"BLOCK\",\"terminatingRuleMatchDetails\":[{\"conditionType\":\"SQL_INJECTION\",\"location\":\"HEADER\",\"matchedData\":[\"10\",\"AND\",\"1\"]}],\"httpSourceName\":\"-\",\"httpSourceId\":\"-\",\"ruleGroupList\":[],\"rateBasedRuleList\":[],\"nonTerminatingMatchingRules\":[],\"httpRequest\":{\"clientIp\":\"89.160.20.156\",\"country\":\"AU\",\"headers\":[{\"name\":\"Host\",\"value\":\"localhost:1989\"},{\"name\":\"User-Agent\",\"value\":\"curl/7.61.1\"},{\"name\":\"Accept\",\"value\":\"*/*\"},{\"name\":\"x-stm-test\",\"value\":\"10 AND 1=1\"}],\"uri\":\"/foo\",\"args\":\"\",\"httpVersion\":\"HTTP/1.1\",\"httpMethod\":\"GET\",\"requestId\":\"rid\"},\"labels\":[{\"name\":\"value\"}]}", + "category": "web", + "type": [ + "access", + "denied" + ], + "kind": "event" + }, + "aws": { + "waf": { + "request": { + "headers": { + "User-Agent": "curl/7.61.1", + "Host": "localhost:1989", + "Accept": "*/*", + "x-stm-test": "10 AND 1=1" + } + }, + "terminating_rule_match_details": [ + { + "conditionType": "SQL_INJECTION", + "location": "HEADER", + "matchedData": [ + "10", + "AND", + "1" + ] + } + ], + "id": "regional/webacl/STMTest/1EXAMPLE-2ARN-3ARN-4ARN-123456EXAMPLE", + "arn": "arn:aws:wafv2:ap-southeast-2:EXAMPLE12345:regional/webacl/STMTest/1EXAMPLE-2ARN-3ARN-4ARN-123456EXAMPLE" + } + } + }, + { + "rule": { + "ruleset": "REGULAR", + "id": "Default_Action" + }, + "source": { + "geo": { + "continent_name": "Europe", + "region_iso_code": "SE-AB", + "city_name": "Tumba", + "country_iso_code": "SE", + "country_name": "Sweden", + "region_name": "Stockholm", + "location": { + "lon": 17.8167, + "lat": 59.2 + } + }, + "as": { + "number": 29518, + "organization": { + "name": "Bredband2 AB" + } + }, + "ip": "89.160.20.156" + }, + "url": { + "path": "/foo" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "http", + "transport": "tcp" + }, + "cloud": { + "region": "us-east-1", + "provider": "aws", + "service": { + "name": "wafv2" + }, + "account": { + "id": "123456789012" + } + }, + "ecs": { + "version": "1.12.0" + }, + "related": { + "ip": [ + "89.160.20.156" + ] + }, + "http": { + "request": { + "method": "GET", + "id": "rid" + }, + "version": "1.1" + }, + "event": { + "action": "ALLOW", + "ingested": "2021-12-09T16:12:01.711626900Z", + "original": "{\"timestamp\":1592357192516,\"formatVersion\":1,\"webaclId\":\"arn:aws:wafv2:us-east-1:123456789012:global/webacl/hello-world/5933d6d9-9dde-js82-v8aw-9ck28nv9\",\"terminatingRuleId\":\"Default_Action\",\"terminatingRuleType\":\"REGULAR\",\"action\":\"ALLOW\",\"terminatingRuleMatchDetails\":[],\"httpSourceName\":\"-\",\"httpSourceId\":\"-\",\"ruleGroupList\":[],\"rateBasedRuleList\":[],\"nonTerminatingMatchingRules\":[{\"ruleId\":\"TestRule\",\"action\":\"COUNT\",\"ruleMatchDetails\":[{\"conditionType\":\"SQL_INJECTION\",\"location\":\"HEADER\",\"matchedData\":[\"10\",\"and\",\"1\"]}]}],\"httpRequest\":{\"clientIp\":\"89.160.20.156\",\"country\":\"US\",\"headers\":[{\"name\":\"Host\",\"value\":\"localhost:1989\"},{\"name\":\"User-Agent\",\"value\":\"curl/7.61.1\"},{\"name\":\"Accept\",\"value\":\"*/*\"},{\"name\":\"foo\",\"value\":\"10 AND 1=1\"}],\"uri\":\"/foo\",\"args\":\"\",\"httpVersion\":\"HTTP/1.1\",\"httpMethod\":\"GET\",\"requestId\":\"rid\"},\"labels\":[{\"name\":\"value\"}]}", + "category": "web", + "type": [ + "access", + "allowed" + ], + "kind": "event" + }, + "aws": { + "waf": { + "request": { + "headers": { + "User-Agent": "curl/7.61.1", + "Host": "localhost:1989", + "Accept": "*/*", + "foo": "10 AND 1=1" + } + }, + "id": "global/webacl/hello-world/5933d6d9-9dde-js82-v8aw-9ck28nv9", + "arn": "arn:aws:wafv2:us-east-1:123456789012:global/webacl/hello-world/5933d6d9-9dde-js82-v8aw-9ck28nv9", + "non_terminating_matching_rules": [ + { + "ruleId": "TestRule", + "action": "COUNT", + "ruleMatchDetails": [ + { + "conditionType": "SQL_INJECTION", + "location": "HEADER", + "matchedData": [ + "10", + "and", + "1" + ] + } + ] + } + ] + } + } + }, + { + "rule": { + "ruleset": "GROUP", + "id": "RG-Reference" + }, + "source": { + "geo": { + "continent_name": "Europe", + "region_iso_code": "SE-AB", + "city_name": "Tumba", + "country_iso_code": "SE", + "country_name": "Sweden", + "region_name": "Stockholm", + "location": { + "lon": 17.8167, + "lat": 59.2 + } + }, + "as": { + "number": 29518, + "organization": { + "name": "Bredband2 AB" + } + }, + "ip": "89.160.20.156" + }, + "url": { + "path": "/foo" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "http", + "transport": "tcp" + }, + "cloud": { + "region": "us-east-1", + "provider": "aws", + "service": { + "name": "wafv2" + }, + "account": { + "id": "123456789012" + } + }, + "ecs": { + "version": "1.12.0" + }, + "related": { + "ip": [ + "89.160.20.156" + ] + }, + "http": { + "request": { + "method": "GET", + "id": "rid" + }, + "version": "1.1" + }, + "event": { + "action": "BLOCK", + "ingested": "2021-12-09T16:12:01.711632900Z", + "original": "{\"timestamp\":1592361810888,\"formatVersion\":1,\"webaclId\":\"arn:aws:wafv2:us-east-1:123456789012:global/webacl/hello-world/5933d6d9-9dde-js82-v8aw-9ck28nv9\",\"terminatingRuleId\":\"RG-Reference\",\"terminatingRuleType\":\"GROUP\",\"action\":\"BLOCK\",\"terminatingRuleMatchDetails\":[{\"conditionType\":\"XSS\",\"location\":\"HEADER\",\"matchedData\":[\"\u003c\",\"frameset\"]}],\"httpSourceName\":\"-\",\"httpSourceId\":\"-\",\"ruleGroupList\":[{\"ruleGroupId\":\"arn:aws:wafv2:us-east-1:123456789012:global/rulegroup/hello-world/c05lb698-1f11-4m41-aef4-99a506d53f4b\",\"terminatingRule\":{\"ruleId\":\"RuleA-XSS\",\"action\":\"BLOCK\",\"ruleMatchDetails\":null},\"nonTerminatingMatchingRules\":[{\"ruleId\":\"RuleB-SQLi\",\"action\":\"COUNT\",\"ruleMatchDetails\":[{\"conditionType\":\"SQL_INJECTION\",\"location\":\"HEADER\",\"matchedData\":[\"10\",\"and\",\"1\"]}]}],\"excludedRules\":null}],\"rateBasedRuleList\":[],\"nonTerminatingMatchingRules\":[],\"httpRequest\":{\"clientIp\":\"89.160.20.156\",\"country\":\"US\",\"headers\":[{\"name\":\"Host\",\"value\":\"localhost:1989\"},{\"name\":\"User-Agent\",\"value\":\"curl/7.61.1\"},{\"name\":\"Accept\",\"value\":\"*/*\"},{\"name\":\"xssfoo\",\"value\":\"\u003cframeset onload=alert(1)\u003e\"},{\"name\":\"bar\",\"value\":\"10 AND 1=1\"}],\"uri\":\"/foo\",\"args\":\"\",\"httpVersion\":\"HTTP/1.1\",\"httpMethod\":\"GET\",\"requestId\":\"rid\"},\"labels\":[{\"name\":\"value\"}]}", + "category": "web", + "type": [ + "access", + "denied" + ], + "kind": "event" + }, + "aws": { + "waf": { + "request": { + "headers": { + "User-Agent": "curl/7.61.1", + "xssfoo": "\u003cframeset onload=alert(1)\u003e", + "bar": "10 AND 1=1", + "Host": "localhost:1989", + "Accept": "*/*" + } + }, + "terminating_rule_match_details": [ + { + "conditionType": "XSS", + "location": "HEADER", + "matchedData": [ + "\u003c", + "frameset" + ] + } + ], + "rule_group_list": [ + { + "nonTerminatingMatchingRules": [ + { + "ruleId": "RuleB-SQLi", + "action": "COUNT", + "ruleMatchDetails": [ + { + "conditionType": "SQL_INJECTION", + "location": "HEADER", + "matchedData": [ + "10", + "and", + "1" + ] + } + ] + } + ], + "ruleGroupId": "arn:aws:wafv2:us-east-1:123456789012:global/rulegroup/hello-world/c05lb698-1f11-4m41-aef4-99a506d53f4b", + "terminatingRule": { + "ruleId": "RuleA-XSS", + "action": "BLOCK" + } + } + ], + "id": "global/webacl/hello-world/5933d6d9-9dde-js82-v8aw-9ck28nv9", + "arn": "arn:aws:wafv2:us-east-1:123456789012:global/webacl/hello-world/5933d6d9-9dde-js82-v8aw-9ck28nv9" + } + } + }, + { + "rule": { + "ruleset": "REGULAR", + "id": "STMTest_SQLi_XSS" + }, + "source": { + "geo": { + "continent_name": "Europe", + "region_iso_code": "SE-AB", + "city_name": "Tumba", + "country_iso_code": "SE", + "country_name": "Sweden", + "region_name": "Stockholm", + "location": { + "lon": 17.8167, + "lat": 59.2 + } + }, + "as": { + "number": 29518, + "organization": { + "name": "Bredband2 AB" + } + }, + "ip": "89.160.20.156" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "http", + "transport": "tcp" + }, + "cloud": { + "region": "ap-southeast-2", + "provider": "aws", + "service": { + "name": "wafv2" + }, + "account": { + "id": "12345" + } + }, + "ecs": { + "version": "1.12.0" + }, + "related": { + "ip": [ + "89.160.20.156" + ] + }, + "http": { + "request": { + "method": "POST", + "id": "null" + }, + "version": "1.1" + }, + "event": { + "action": "BLOCK", + "ingested": "2021-12-09T16:12:01.711639Z", + "original": "{\"timestamp\":1576280412771,\"formatVersion\":1,\"webaclId\":\"arn:aws:wafv2:ap-southeast-2:12345:regional/webacl/test/111\",\"terminatingRuleId\":\"STMTest_SQLi_XSS\",\"terminatingRuleType\":\"REGULAR\",\"action\":\"BLOCK\",\"terminatingRuleMatchDetails\":[{\"conditionType\":\"SQL_INJECTION\",\"location\":\"UNKNOWN\",\"matchedData\":[\"10\",\"AND\",\"1\"]}],\"httpSourceName\":\"ALB\",\"httpSourceId\":\"alb\",\"ruleGroupList\":[],\"rateBasedRuleList\":[],\"nonTerminatingMatchingRules\":[],\"requestHeadersInserted\":null,\"responseCodeSent\":null,\"httpRequest\":{\"clientIp\":\"89.160.20.156\",\"country\":\"AU\",\"headers\":[],\"uri\":\"\",\"args\":\"\",\"httpVersion\":\"HTTP/1.1\",\"httpMethod\":\"POST\",\"requestId\":\"null\"},\"labels\":[{\"name\":\"value\"}]}", + "category": "web", + "type": [ + "access", + "denied" + ], + "kind": "event" + }, + "aws": { + "waf": { + "terminating_rule_match_details": [ + { + "conditionType": "SQL_INJECTION", + "location": "UNKNOWN", + "matchedData": [ + "10", + "AND", + "1" + ] + } + ], + "id": "regional/webacl/test/111", + "source": { + "name": "ALB", + "id": "alb" + }, + "arn": "arn:aws:wafv2:ap-southeast-2:12345:regional/webacl/test/111" + } + } + } + ] +} \ No newline at end of file diff --git a/test/packages/aws/data_stream/waf/agent/stream/aws-s3.yml.hbs b/test/packages/aws/data_stream/waf/agent/stream/aws-s3.yml.hbs new file mode 100644 index 0000000000..aac824c6be --- /dev/null +++ b/test/packages/aws/data_stream/waf/agent/stream/aws-s3.yml.hbs @@ -0,0 +1,52 @@ +queue_url: {{queue_url}} +content_type: application/json +{{#if credential_profile_name}} +credential_profile_name: {{credential_profile_name}} +{{/if}} +{{#if shared_credential_file}} +shared_credential_file: {{shared_credential_file}} +{{/if}} +{{#if visibility_timeout}} +visibility_timeout: {{visibility_timeout}} +{{/if}} +{{#if api_timeout}} +api_timeout: {{api_timeout}} +{{/if}} +{{#if max_number_of_messages}} +max_number_of_messages: {{max_number_of_messages}} +{{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} +{{#if access_key_id}} +access_key_id: {{access_key_id}} +{{/if}} +{{#if secret_access_key}} +secret_access_key: {{secret_access_key}} +{{/if}} +{{#if session_token}} +session_token: {{session_token}} +{{/if}} +{{#if role_arn}} +role_arn: {{role_arn}} +{{/if}} +{{#if fips_enabled}} +fips_enabled: {{fips_enabled}} +{{/if}} +{{#if proxy_url }} +proxy_url: {{proxy_url}} +{{/if}} +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} \ No newline at end of file diff --git a/test/packages/aws/data_stream/waf/agent/stream/log.yml.hbs b/test/packages/aws/data_stream/waf/agent/stream/log.yml.hbs new file mode 100644 index 0000000000..de93a0f025 --- /dev/null +++ b/test/packages/aws/data_stream/waf/agent/stream/log.yml.hbs @@ -0,0 +1,19 @@ +paths: + {{#each paths as |path i|}} +- {{path}} + {{/each}} +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +exclude_files: [".gz$"] +{{#if processors}} +processors: +{{processors}} +{{/if}} \ No newline at end of file diff --git a/test/packages/aws/data_stream/waf/elasticsearch/ingest_pipeline/default.yml b/test/packages/aws/data_stream/waf/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 0000000000..79bcd949ca --- /dev/null +++ b/test/packages/aws/data_stream/waf/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,204 @@ +--- +description: "Pipeline for WAF logs" +processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' +- set: + field: ecs.version + value: '1.12.0' +- set: + field: event.category + value: web +- append: + field: event.type + value: access +- rename: + field: message + target_field: event.original + ignore_missing: true +- json: + field: event.original + target_field: json +- date: + field: json.timestamp' + target_field: '@timestamp' + ignore_failure: true + formats: + - UNIX_MS +- rename: + field: json.httpRequest.clientIp + target_field: source.ip + ignore_missing: true +- geoip: + field: source.ip + target_field: source.geo + ignore_missing: true +- rename: + field: json.httpRequest.country + target_field: source.geo.country_iso_code + ignore_missing: true + if: ctx.source?.geo.country_iso_code == null +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true +- rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true +- rename: + field: json.ClientASN + target_field: source.as.number + ignore_missing: true + if: ctx?.source?.as?.number == null +- rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true +- rename: + field: json.httpRequest.requestId + target_field: http.request.id + ignore_missing: true +- rename: + field: json.httpRequest.httpMethod + target_field: http.request.method + ignore_missing: true +- dissect: + field: json.httpRequest.httpVersion + pattern: "%{network.protocol}/%{http.version}" + ignore_failure: true +- lowercase: + field: network.protocol + ignore_missing: true +- set: + field: network.transport + value: tcp + if: ctx?.network?.protocol != null && ctx?.network?.protocol == 'http' +- rename: + field: json.httpRequest.args + target_field: url.query + ignore_missing: true +- rename: + field: json.httpRequest.uri + target_field: url.path + ignore_missing: true +- rename: + field: json.terminatingRuleMatchDetails + target_field: aws.waf.terminating_rule_match_details + ignore_missing: true +- rename: + field: json.ruleGroupList + target_field: aws.waf.rule_group_list + ignore_missing: true +- rename: + field: json.rateBasedRuleList + target_field: aws.waf.rate_based_rule_list + ignore_missing: true +- rename: + field: json.nonTerminatingMatchingRules + target_field: aws.waf.non_terminating_matching_rules + ignore_missing: true +- script: + lang: painless + source: >- + if (ctx.json.httpRequest.headers != null) { + ctx.aws.waf.request = new HashMap(); + ctx.aws.waf.request.headers = new HashMap(); + for (def i = 0; i < ctx.json.httpRequest.headers.length; i++) { + ctx.aws.waf.request.headers[ctx.json.httpRequest.headers[i].name] = ctx.json.httpRequest.headers[i].value; + } + } + ignore_failure: true +- rename: + field: json.action + target_field: event.action + ignore_missing: true +- append: + field: related.ip + value: '{{source.ip}}' + allow_duplicates: false + if: ctx.source?.ip != null +- set: + field: cloud.provider + value: aws +- set: + field: event.kind + value: event +- append: + field: event.type + value: allowed + if: ctx.event.action == "ALLOW" +- append: + field: event.type + value: denied + if: ctx.event.action == "BLOCK" +- rename: + field: json.webaclId + target_field: aws.waf.arn + ignore_missing: true +- dissect: + field: aws.waf.arn + pattern: "arn:%{}:%{cloud.service.name}:%{cloud.region}:%{cloud.account.id}:%{aws.waf.id}" + ignore_failure: true + ignore_missing: true +- rename: + field: json.terminatingRuleId + target_field: rule.id + ignore_missing: true +- rename: + field: json.terminatingRuleType + target_field: rule.ruleset + ignore_missing: true +- rename: + field: json.httpSourceName + target_field: aws.waf.source.name + ignore_missing: true +- rename: + field: json.httpSourceId + target_field: aws.waf.source.id + ignore_missing: true + + # + # Remove temporary fields + # +- remove: + field: json + ignore_missing: true +- script: + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null || v == "" || v == "-" || ((v instanceof List || v instanceof Map) && v.isEmpty())); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx); +- remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true +on_failure: + - set: + field: 'error.message' + value: '{{ _ingest.on_failure_message }}' diff --git a/test/packages/aws/data_stream/waf/fields/agent.yml b/test/packages/aws/data_stream/waf/fields/agent.yml new file mode 100644 index 0000000000..da4e652c53 --- /dev/null +++ b/test/packages/aws/data_stream/waf/fields/agent.yml @@ -0,0 +1,198 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + diff --git a/test/packages/aws/data_stream/waf/fields/base-fields.yml b/test/packages/aws/data_stream/waf/fields/base-fields.yml new file mode 100644 index 0000000000..417828aa0b --- /dev/null +++ b/test/packages/aws/data_stream/waf/fields/base-fields.yml @@ -0,0 +1,20 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. +- name: event.module + type: constant_keyword + description: Event module + value: aws +- name: event.dataset + type: constant_keyword + description: Event dataset + value: aws.waf diff --git a/test/packages/aws/data_stream/waf/fields/ecs.yml b/test/packages/aws/data_stream/waf/fields/ecs.yml new file mode 100644 index 0000000000..cf3ab8d9b7 --- /dev/null +++ b/test/packages/aws/data_stream/waf/fields/ecs.yml @@ -0,0 +1,60 @@ +- external: ecs + name: source.address +- external: ecs + name: source.ip +- external: ecs + name: cloud.provider +- external: ecs + name: ecs.version +- external: ecs + name: error.message +- external: ecs + name: event.action +- external: ecs + name: event.id +- external: ecs + name: event.kind +- external: ecs + name: event.outcome +- external: ecs + name: http.request.method +- external: ecs + name: http.version +- external: ecs + name: http.request.id +- external: ecs + name: related.ip +- external: ecs + name: tags +- external: ecs + name: url.path +- external: ecs + name: url.query +- external: ecs + name: source.as.number +- external: ecs + name: source.as.organization.name +- external: ecs + name: source.geo.city_name +- external: ecs + name: source.geo.continent_name +- external: ecs + name: source.geo.country_iso_code +- external: ecs + name: source.geo.country_name +- description: Longitude and latitude. + level: core + name: source.geo.location + type: geo_point +- external: ecs + name: source.geo.region_iso_code +- external: ecs + name: source.geo.region_name +- external: ecs + name: network.protocol +- external: ecs + name: network.transport +- external: ecs + name: rule.id +- external: ecs + name: rule.ruleset diff --git a/test/packages/aws/data_stream/waf/fields/fields.yml b/test/packages/aws/data_stream/waf/fields/fields.yml new file mode 100644 index 0000000000..b4a9343da9 --- /dev/null +++ b/test/packages/aws/data_stream/waf/fields/fields.yml @@ -0,0 +1,39 @@ +- name: aws.waf + type: group + fields: + - name: arn + type: keyword + description: | + AWS ARN of ACL + - name: id + type: keyword + description: | + ID of ACL + - name: non_terminating_matching_rules + type: nested + description: | + The list of non-terminating rules in the rule group that match the request. These are always COUNT rules (non-terminating rules that match) + - name: rate_based_rule_list + type: nested + description: | + The list of rate-based rules that acted on the request. + - name: request.headers + type: flattened + description: | + List of request headers + - name: rule_group_list + type: nested + description: | + The list of rule groups that acted on this request. + - name: source.id + type: keyword + description: | + The source ID. This field shows the ID of the associated resource. + - name: source.name + type: keyword + description: | + The source of the request. Possible values: CF for Amazon CloudFront, APIGW for Amazon API Gateway, ALB for Application Load Balancer, and APPSYNC for AWS AppSync. + - name: terminating_rule_match_details + type: nested + description: | + Detailed information about the terminating rule that matched the request. A terminating rule has an action that ends the inspection process against a web request. Possible actions for a terminating rule are ALLOW and BLOCK. This is only populated for SQL injection and cross-site scripting (XSS) match rule statements. As with all rule statements that inspect for more than one thing, AWS WAF applies the action on the first match and stops inspecting the web request. A web request with a terminating action could contain other threats, in addition to the one reported in the log. diff --git a/test/packages/aws/data_stream/waf/manifest.yml b/test/packages/aws/data_stream/waf/manifest.yml new file mode 100644 index 0000000000..9abff552b5 --- /dev/null +++ b/test/packages/aws/data_stream/waf/manifest.yml @@ -0,0 +1,70 @@ +title: AWS WAF logs +type: logs +streams: + - input: aws-s3 + template_path: aws-s3.yml.hbs + title: AWS WAF logs + description: Collect AWS WAF logs using s3 input + vars: + - name: visibility_timeout + type: text + title: Visibility Timeout + multi: false + required: false + show_user: false + description: The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours. + - name: api_timeout + type: text + title: API Timeout + multi: false + required: false + show_user: false + description: The maximum duration of AWS API can take. The maximum is half of the visibility timeout value. + - name: queue_url + type: text + title: Queue URL + multi: false + required: true + show_user: true + description: URL of the AWS SQS queue that messages will be received from. + - name: fips_enabled + type: bool + title: Enable S3 FIPS + default: false + multi: false + required: false + show_user: false + description: Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - aws-waf + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: max_number_of_messages + type: integer + title: Maximum Concurrent SQS Messages + description: The maximum number of SQS messages that can be inflight at any time. + default: 5 + required: false + show_user: false diff --git a/test/packages/aws/data_stream/waf/sample_event.json b/test/packages/aws/data_stream/waf/sample_event.json new file mode 100644 index 0000000000..16cdada741 --- /dev/null +++ b/test/packages/aws/data_stream/waf/sample_event.json @@ -0,0 +1,94 @@ +{ + "@timestamp": "2021-11-25T14:25:25.000Z", + "data_stream": { + "namespace": "default", + "type": "logs", + "dataset": "aws.waf" + }, + "rule": { + "ruleset": "REGULAR", + "id": "STMTest_SQLi_XSS" + }, + "source": { + "geo": { + "continent_name": "Oceania", + "country_name": "Australia", + "location": { + "lon": 143.2104, + "lat": -33.494 + }, + "country_iso_code": "AU" + }, + "as": { + "number": 13335, + "organization": { + "name": "Cloudflare, Inc." + } + }, + "ip": "1.1.1.1" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "http", + "transport": "tcp" + }, + "cloud": { + "region": "ap-southeast-2", + "provider": "aws", + "service": { + "name": "wafv2" + }, + "account": { + "id": "12345" + } + }, + "ecs": { + "version": "1.12.0" + }, + "related": { + "ip": [ + "1.1.1.1" + ] + }, + "http": { + "request": { + "method": "POST", + "id": "null" + }, + "version": "1.1" + }, + "event": { + "action": "BLOCK", + "ingested": "2021-10-11T15:00:35.544818361Z", + "original": "{\"timestamp\":1576280412771,\"formatVersion\":1,\"webaclId\":\"arn:aws:wafv2:ap-southeast-2:12345:regional/webacl/test/111\",\"terminatingRuleId\":\"STMTest_SQLi_XSS\",\"terminatingRuleType\":\"REGULAR\",\"action\":\"BLOCK\",\"terminatingRuleMatchDetails\":[{\"conditionType\":\"SQL_INJECTION\",\"location\":\"UNKNOWN\",\"matchedData\":[\"10\",\"AND\",\"1\"]}],\"httpSourceName\":\"ALB\",\"httpSourceId\":\"alb\",\"ruleGroupList\":[],\"rateBasedRuleList\":[],\"nonTerminatingMatchingRules\":[],\"requestHeadersInserted\":null,\"responseCodeSent\":null,\"httpRequest\":{\"clientIp\":\"1.1.1.1\",\"country\":\"AU\",\"headers\":[],\"uri\":\"\",\"args\":\"\",\"httpVersion\":\"HTTP/1.1\",\"httpMethod\":\"POST\",\"requestId\":\"null\"},\"labels\":[{\"name\":\"value\"}]}", + "category": "web", + "type": [ + "access", + "denied" + ], + "kind": "event" + }, + "aws": { + "waf": { + "terminating_rule_match_details": [ + { + "conditionType": "SQL_INJECTION", + "location": "UNKNOWN", + "matchedData": [ + "10", + "AND", + "1" + ] + } + ], + "id": "regional/webacl/test/111", + "source": { + "name": "ALB", + "id": "alb" + }, + "arn": "arn:aws:wafv2:ap-southeast-2:12345:regional/webacl/test/111" + } + } +} \ No newline at end of file diff --git a/test/packages/aws/docs/README.md b/test/packages/aws/docs/README.md index e36b499695..c40242cdf6 100644 --- a/test/packages/aws/docs/README.md +++ b/test/packages/aws/docs/README.md @@ -15,6 +15,20 @@ AWS credentials are required for running AWS integration. * *endpoint*: URL of the entry point for an AWS web service. * *role_arn*: AWS IAM Role to assume. +#### Data stream specific configuration parameters +* *latency*: Some AWS services send monitoring metrics to CloudWatch with a +latency to process larger than Metricbeat collection period. This will cause +data points missing or none get collected by Metricbeat. In this case, please +specify a latency parameter so collection start time and end time will be +shifted by the given latency amount. +* *period*: How often the data stream is executed. +* *regions*: Specify which AWS regions to query metrics from. If the `regions` +is not set in the config, then by default, the integration will query metrics +from all available AWS regions. If `endpoint` is specified, `regions` becomes a +required config parameter. +* *tags_filter*: Tag key value pairs from aws resources. A tag is a label that +user assigns to an AWS resource. + ### Credential Types There are three types of AWS credentials can be used: access keys, temporary security credentials and IAM role ARN. @@ -61,10 +75,14 @@ temporary credentials. Please see for more details. ### Supported Formats -1. Use `access_key_id`, `secret_access_key` and/or `session_token` directly -2. Use `role_arn`: If `access_key_id` and `secret_access_key` are not given, -then the package will check for `role_arn`. `role_arn` is used to specify which - AWS IAM role to assume for generating temporary credentials. +1. Use access keys: Access keys include `access_key_id`, `secret_access_key` +and/or `session_token`. +2. Use `role_arn`: `role_arn` is used to specify which AWS IAM role to assume +for generating temporary credentials. If `role_arn` is given, the package will +check if access keys are given. If not, the package will check for credential +profile name. If neither is given, default credential profile will be used. +Please make sure credentials are given under either a credential profile or +access keys. 3. Use `credential_profile_name` and/or `shared_credential_file`: If `access_key_id`, `secret_access_key` and `role_arn` are all not given, then the package will check for `credential_profile_name`. If you use different diff --git a/test/packages/aws/docs/billing.md b/test/packages/aws/docs/billing.md index ec22eaf702..b83943364c 100644 --- a/test/packages/aws/docs/billing.md +++ b/test/packages/aws/docs/billing.md @@ -29,17 +29,29 @@ An example event for `billing` looks as following: }, "aws": { "billing": { - "metrics": { - "EstimatedCharges": { - "max": 1625.41 - } + "Currency": "USD", + "EstimatedCharges": 39.26, + "ServiceName": "AmazonEKS", + "AmortizedCost": { + "amount": 51.6, + "unit": "USD" + }, + "BlendedCost": { + "amount": 51.6, + "unit": "USD" + }, + "NormalizedUsageAmount": { + "amount": 672, + "unit": "N/A" + }, + "UnblendedCost": { + "amount": 51.6, + "unit": "USD" + }, + "UsageQuantity": { + "amount": 168, + "unit": "N/A" } - }, - "cloudwatch": { - "namespace": "AWS/Billing" - }, - "dimensions": { - "Currency": "USD" } }, "service": { @@ -66,7 +78,7 @@ An example event for `billing` looks as following: | aws.billing.BlendedCost.amount | Blended cost amount. | double | | aws.billing.BlendedCost.unit | Blended cost unit. | keyword | | aws.billing.Currency | Currency name. | keyword | -| aws.billing.EstimatedCharges.max | Maximum estimated charges for AWS acccount. | long | +| aws.billing.EstimatedCharges | Maximum estimated charges for AWS acccount. | long | | aws.billing.NormalizedUsageAmount.amount | Normalized usage amount. | double | | aws.billing.NormalizedUsageAmount.unit | Normalized usage amount unit. | keyword | | aws.billing.ServiceName | AWS service name. | keyword | @@ -81,8 +93,11 @@ An example event for `billing` looks as following: | aws.billing.start_date | Start date for retrieving AWS costs. | keyword | | aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword | | aws.dimensions.\* | Metric dimensions. | object | +| aws.linked_account.id | ID used to identify linked account. | keyword | +| aws.linked_account.name | Name or alias used to identify linked account. | keyword | | aws.s3.bucket.name | Name of a S3 bucket. | keyword | | aws.tags.\* | Tag key value pairs from aws resources. | object | +| cloud | Fields related to the cloud or infrastructure the events are coming from. | group | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | | cloud.availability_zone | Availability zone in which this host is running. | keyword | @@ -100,7 +115,11 @@ An example event for `billing` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. | keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| error | These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. | group | +| error.message | Error message. | match_only_text | +| event.dataset | Event dataset | constant_keyword | +| event.module | Event module | constant_keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -117,5 +136,5 @@ An example event for `billing` looks as following: | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| service.type | Service type | keyword | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | diff --git a/test/packages/aws/docs/cloudtrail.md b/test/packages/aws/docs/cloudtrail.md index 7edd7c85a8..1512ae2fd1 100644 --- a/test/packages/aws/docs/cloudtrail.md +++ b/test/packages/aws/docs/cloudtrail.md @@ -59,7 +59,7 @@ events for the account. If user creates a trail, it delivers those events as log | cloud.machine.type | Machine type of the host machine. | keyword | | cloud.project.id | Name of the project in Google Cloud. | keyword | | cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | +| cloud.region | Region in which this host, resource, or service is located. | keyword | | container.id | Unique container id. | keyword | | container.image.name | Name of the image the container was built on. | keyword | | container.labels | Image labels. | object | @@ -67,13 +67,17 @@ events for the account. If user creates a trail, it delivers those events as log | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| error.message | Error message. | text | -| event.action | The action captured by the event. | keyword | -| event.ingested | Timestamp when an event arrived in the central data store. | date | -| event.kind | Event kind (e.g. event, alert, metric, state, pipeline_error, signal) | keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity. | keyword | -| event.provider | Source of the event. | keyword | -| event.type | Event severity (e.g. info, error) | keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| error.message | Error message. | match_only_text | +| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | +| event.created | event.created contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from @timestamp in that @timestamp typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, @timestamp should be used. | date | +| event.dataset | Event dataset | constant_keyword | +| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | +| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data coming in at a regular interval or not. | keyword | +| event.module | Event module | constant_keyword | +| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | +| event.provider | Source of the event. Event transports such as Syslog or the Windows Event Log typically mention the source of an event. It can be the name of the software that generated the event (e.g. Sysmon, httpd), or of a subsystem of the operating system (kernel, Microsoft-Windows-Security-Auditing). | keyword | +| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | | file.hash.md5 | MD5 hash. | keyword | | file.hash.sha1 | SHA1 hash. | keyword | | file.hash.sha256 | SHA256 hash. | keyword | @@ -97,9 +101,9 @@ events for the account. If user creates a trail, it delivers those events as log | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| related.hash | All the hashes seen on your event. | keyword | -| related.user | All the user names seen on your event. | keyword | -| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the .address field. | keyword | +| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | +| related.user | All the user names or other user identifiers seen on the event. | keyword | +| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | | source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | | source.as.organization.name | Organization name. | keyword | | source.geo.city_name | City name. | keyword | @@ -110,6 +114,7 @@ events for the account. If user creates a trail, it delivers those events as log | source.geo.region_iso_code | Region ISO code. | keyword | | source.geo.region_name | Region name. | keyword | | source.ip | IP address of the source (IPv4 or IPv6). | ip | +| tags | List of keywords used to tag each event. | keyword | | user.changes.name | Short name or login of the user. | keyword | | user.id | Unique identifier of the user. | keyword | | user.name | Short name or login of the user. | keyword | @@ -123,3 +128,93 @@ events for the account. If user creates a trail, it delivers those events as log | user_agent.os.version | Operating system version as a raw string. | keyword | | user_agent.version | Version of the user agent. | keyword | + +An example event for `cloudtrail` looks as following: + +```json +{ + "data_stream": { + "namespace": "default", + "type": "logs", + "dataset": "aws.cloudtrail" + }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "tags": [ + "preserve_original_event" + ], + "cloud": { + "region": "us-east-1", + "account": { + "id": "123456789012" + } + }, + "@timestamp": "2020-01-08T20:53:12.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "user": [ + "Alice", + "Bob", + "Robert" + ] + }, + "event": { + "ingested": "2021-10-05T23:06:12.229540200Z", + "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EX_PRINCIPAL_ID\",\"arn\":\"arn:aws:iam::123456789012:user/Alice\",\"accountId\":\"123456789012\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Alice\"},\"eventTime\":\"2020-01-08T20:53:12Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"UpdateUser\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46\",\"requestParameters\":{\"userName\":\"Bob\",\"newUserName\":\"Robert\"},\"responseElements\":null,\"requestID\":\"3a6b3260-739d-465e-9406-bcEXAMPLE\",\"eventID\":\"9150d546-3564-4262-8e62-110EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"123456789012\"}", + "provider": "iam.amazonaws.com", + "created": "2020-01-08T20:53:12.000Z", + "kind": "event", + "action": "UpdateUser", + "id": "9150d546-3564-4262-8e62-110EXAMPLE", + "type": [ + "user", + "change" + ], + "category": [ + "iam" + ], + "outcome": "success" + }, + "aws": { + "cloudtrail": { + "event_version": "1.05", + "flattened": { + "request_parameters": { + "userName": "Bob", + "newUserName": "Robert" + } + }, + "user_identity": { + "access_key_id": "EXAMPLE_KEY_ID", + "type": "IAMUser", + "arn": "arn:aws:iam::123456789012:user/Alice" + }, + "event_type": "AwsApiCall", + "recipient_account_id": "123456789012", + "request_parameters": "{newUserName=Robert, userName=Bob}" + } + }, + "user": { + "name": "Alice", + "changes": { + "name": "Robert" + }, + "id": "EX_PRINCIPAL_ID", + "target": { + "name": "Bob" + } + }, + "user_agent": { + "name": "aws-cli", + "original": "aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46", + "device": { + "name": "Spider" + }, + "version": "1.16.310" + } +} +``` diff --git a/test/packages/aws/docs/cloudwatch.md b/test/packages/aws/docs/cloudwatch.md index 8e43e471dc..d6ab5bcca5 100644 --- a/test/packages/aws/docs/cloudwatch.md +++ b/test/packages/aws/docs/cloudwatch.md @@ -29,6 +29,10 @@ setup already. | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| error.message | Error message. | match_only_text | +| event.dataset | Event dataset | constant_keyword | +| event.module | Event module | constant_keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -45,8 +49,37 @@ setup already. | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| tags | List of keywords used to tag each event. | keyword | +An example event for `cloudwatch` looks as following: + +```json +{ + "@timestamp": "2020-02-20T07:02:37.000Z", + "data_stream": { + "namespace": "default", + "type": "logs", + "dataset": "aws.cloudwatch_logs" + }, + "ecs": { + "version": "1.12.0" + }, + "event": { + "ingested": "2021-07-19T21:47:04.696803300Z", + "original": "2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 ec2net: [get_meta] Trying to get http://169.254.169.254/latest/meta-data/network/interfaces/macs/12:e2:a9:95:8b:97/local-ipv4s" + }, + "aws": { + "cloudwatch": { + "message": "ip-172-31-81-156 ec2net: [get_meta] Trying to get http://169.254.169.254/latest/meta-data/network/interfaces/macs/12:e2:a9:95:8b:97/local-ipv4s" + } + }, + "tags": [ + "preserve_original_event" + ] +} +``` + ## Metrics An example event for `cloudwatch` looks as following: @@ -56,7 +89,7 @@ An example event for `cloudwatch` looks as following: "@timestamp": "2020-05-28T17:17:02.812Z", "event": { "duration": 14119105951, - "dataset": "aws.cloudwatch", + "dataset": "aws.cloudwatch_metrics", "module": "aws" }, "ecs": { @@ -117,6 +150,7 @@ An example event for `cloudwatch` looks as following: | aws.dimensions.\* | Metric dimensions. | object | | aws.s3.bucket.name | Name of a S3 bucket. | keyword | | aws.tags.\* | Tag key value pairs from aws resources. | object | +| cloud | Fields related to the cloud or infrastructure the events are coming from. | group | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | | cloud.availability_zone | Availability zone in which this host is running. | keyword | @@ -134,7 +168,11 @@ An example event for `cloudwatch` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. | keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| error | These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. | group | +| error.message | Error message. | match_only_text | +| event.dataset | Event dataset | constant_keyword | +| event.module | Event module | constant_keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -151,4 +189,4 @@ An example event for `cloudwatch` looks as following: | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| service.type | Service type | keyword | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | diff --git a/test/packages/aws/docs/dynamodb.md b/test/packages/aws/docs/dynamodb.md index 42fc2b596c..f9c97eacde 100644 --- a/test/packages/aws/docs/dynamodb.md +++ b/test/packages/aws/docs/dynamodb.md @@ -103,6 +103,7 @@ An example event for `dynamodb` looks as following: | aws.dynamodb.metrics.WriteThrottleEvents.sum | Requests to DynamoDB that exceed the provisioned write capacity units for a table or a global secondary index. | long | | aws.s3.bucket.name | Name of a S3 bucket. | keyword | | aws.tags.\* | Tag key value pairs from aws resources. | object | +| cloud | Fields related to the cloud or infrastructure the events are coming from. | group | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | | cloud.availability_zone | Availability zone in which this host is running. | keyword | @@ -112,7 +113,7 @@ An example event for `dynamodb` looks as following: | cloud.machine.type | Machine type of the host machine. | keyword | | cloud.project.id | Name of the project in Google Cloud. | keyword | | cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | +| cloud.region | Region in which this host, resource, or service is located. | keyword | | container.id | Unique container id. | keyword | | container.image.name | Name of the image the container was built on. | keyword | | container.labels | Image labels. | object | @@ -120,7 +121,11 @@ An example event for `dynamodb` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. | keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| error | These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. | group | +| error.message | Error message. | match_only_text | +| event.dataset | Event dataset | constant_keyword | +| event.module | Event module | constant_keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -137,4 +142,4 @@ An example event for `dynamodb` looks as following: | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| service.type | Service type | keyword | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | diff --git a/test/packages/aws/docs/ebs.md b/test/packages/aws/docs/ebs.md index e4529dffcd..ad2eb31ea5 100644 --- a/test/packages/aws/docs/ebs.md +++ b/test/packages/aws/docs/ebs.md @@ -95,6 +95,7 @@ An example event for `ebs` looks as following: | aws.ebs.metrics.VolumeWriteOps.avg | The total number of write operations in a specified period of time. | double | | aws.s3.bucket.name | Name of a S3 bucket. | keyword | | aws.tags.\* | Tag key value pairs from aws resources. | object | +| cloud | Fields related to the cloud or infrastructure the events are coming from. | group | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | | cloud.availability_zone | Availability zone in which this host is running. | keyword | @@ -104,7 +105,7 @@ An example event for `ebs` looks as following: | cloud.machine.type | Machine type of the host machine. | keyword | | cloud.project.id | Name of the project in Google Cloud. | keyword | | cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | +| cloud.region | Region in which this host, resource, or service is located. | keyword | | container.id | Unique container id. | keyword | | container.image.name | Name of the image the container was built on. | keyword | | container.labels | Image labels. | object | @@ -112,7 +113,11 @@ An example event for `ebs` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. | keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| error | These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. | group | +| error.message | Error message. | match_only_text | +| event.dataset | Event dataset | constant_keyword | +| event.module | Event module | constant_keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -129,4 +134,4 @@ An example event for `ebs` looks as following: | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| service.type | Service type | keyword | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | diff --git a/test/packages/aws/docs/ec2.md b/test/packages/aws/docs/ec2.md index 56e688b5dc..98343270b5 100644 --- a/test/packages/aws/docs/ec2.md +++ b/test/packages/aws/docs/ec2.md @@ -29,6 +29,10 @@ and `process.name`. For logs from other services, please use `cloudwatch` datase | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| error.message | Error message. | match_only_text | +| event.dataset | Event dataset | constant_keyword | +| event.module | Event module | constant_keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -45,9 +49,43 @@ and `process.name`. For logs from other services, please use `cloudwatch` datase | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | | process.name | Process name. | keyword | +| tags | List of keywords used to tag each event. | keyword | +An example event for `ec2` looks as following: + +```json +{ + "data_stream": { + "namespace": "default", + "type": "logs", + "dataset": "aws.ec2_logs" + }, + "process": { + "name": "systemd" + }, + "@timestamp": "2020-02-20T07:01:01.000Z", + "ecs": { + "version": "1.12.0" + }, + "event": { + "ingested": "2021-07-19T21:47:04.871450600Z", + "original": "2020-02-20T07:01:01.000Z Feb 20 07:01:01 ip-172-31-81-156 systemd: Stopping User Slice of root." + }, + "aws": { + "ec2": { + "ip_address": "ip-172-31-81-156" + } + }, + "message": "Stopping User Slice of root.", + "tags": [ + "preserve_original_event" + ] +} +``` + ## Metrics An example event for `ec2` looks as following: @@ -138,7 +176,7 @@ An example event for `ec2` looks as following: "event": { "module": "aws", "duration": 23217499283, - "dataset": "aws.ec2" + "dataset": "aws.ec2_metrics" }, "metricset": { "period": 300000, @@ -213,6 +251,7 @@ An example event for `ec2` looks as following: | aws.ec2.status.check_failed_system | Reports whether the instance has passed the system status check in the last minute. | long | | aws.s3.bucket.name | Name of a S3 bucket. | keyword | | aws.tags.\* | Tag key value pairs from aws resources. | object | +| cloud | Fields related to the cloud or infrastructure the events are coming from. | group | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | | cloud.availability_zone | Availability zone in which this host is running. | keyword | @@ -230,7 +269,11 @@ An example event for `ec2` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. | keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| error | These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. | group | +| error.message | Error message. | match_only_text | +| event.dataset | Event dataset | constant_keyword | +| event.module | Event module | constant_keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.cpu.pct | Percent CPU used. This value is normalized by the number of CPU cores and it ranges from 0 to 1. | scaled_float | @@ -254,4 +297,4 @@ An example event for `ec2` looks as following: | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| service.type | Service type | keyword | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | diff --git a/test/packages/aws/docs/elb.md b/test/packages/aws/docs/elb.md index 5638b1e77b..61e6f6b3a6 100644 --- a/test/packages/aws/docs/elb.md +++ b/test/packages/aws/docs/elb.md @@ -65,9 +65,13 @@ For network load balancer, please follow [enable access log for network load bal | data_stream.type | Data stream type. | constant_keyword | | destination.bytes | Bytes sent from the destination to the source. | long | | destination.domain | Destination domain. | keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| error.message | Error message. | match_only_text | | event.category | Event category (e.g. database) | keyword | +| event.dataset | Event dataset | constant_keyword | | event.end | event.end contains the date when the event ended or when the activity was last observed. | date | | event.kind | Event kind (e.g. event, alert, metric, state, pipeline_error, sig | keyword | +| event.module | Event module | constant_keyword | | event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. | keyword | | event.start | event.start contains the date when the event started or when the activity was first observed. | date | | host.architecture | Operating system architecture. | keyword | @@ -102,10 +106,129 @@ For network load balancer, please follow [enable access log for network load bal | source.geo.region_name | Region name. | keyword | | source.ip | IP address of the source. | ip | | source.port | Port of the source. | keyword | +| tags | List of keywords used to tag each event. | keyword | | tracing.trace.id | Unique identifier of the trace. | keyword | +| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | +| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | +| url.path | Path of the request, such as "/search". | wildcard | +| url.port | Port of the request, such as 443. | long | +| url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | +| user_agent.device.name | Name of the device. | keyword | +| user_agent.name | Name of the user agent. | keyword | | user_agent.original | Unparsed user_agent string. | keyword | +| user_agent.version | Version of the user agent. | keyword | +An example event for `elb` looks as following: + +```json +{ + "data_stream": { + "namespace": "default", + "type": "logs", + "dataset": "aws.elb_logs" + }, + "tracing": { + "trace": { + "id": "Root=1-58337262-36d228ad5d99923122bbe354" + } + }, + "source": { + "port": "2817", + "ip": "192.168.131.39" + }, + "url": { + "path": "/", + "original": "http://www.example.com:80/", + "scheme": "http", + "port": 80, + "domain": "www.example.com" + }, + "tags": [ + "preserve_original_event" + ], + "cloud": { + "provider": "aws" + }, + "@timestamp": "2018-07-02T22:23:00.186Z", + "ecs": { + "version": "1.12.0" + }, + "http": { + "request": { + "method": "get", + "body": { + "bytes": 34 + } + }, + "version": "1.1", + "response": { + "body": { + "bytes": 366 + }, + "status_code": 200 + } + }, + "event": { + "ingested": "2021-07-19T21:47:05.084930900Z", + "original": "http 2018-07-02T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 192.168.131.39:2817 10.0.0.1:80 0.000 0.001 0.000 200 200 34 366 \"GET http://www.example.com:80/ HTTP/1.1\" \"curl/7.46.0\" - - arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 \"Root=1-58337262-36d228ad5d99923122bbe354\" \"-\" \"-\" 0 2018-07-02T22:22:48.364000Z \"forward,redirect\" \"-\" \"-\" \"10.0.0.1:80\" \"200\" \"-\" \"-\"", + "kind": "event", + "start": "2018-07-02T22:22:48.364000Z", + "end": "2018-07-02T22:23:00.186Z", + "category": "web", + "outcome": "success" + }, + "aws": { + "elb": { + "trace_id": "Root=1-58337262-36d228ad5d99923122bbe354", + "matched_rule_priority": "0", + "type": "http", + "request_processing_time": { + "sec": 0.0 + }, + "response_processing_time": { + "sec": 0.0 + }, + "target_port": [ + "10.0.0.1:80" + ], + "protocol": "http", + "target_status_code": [ + "200" + ], + "name": "app/my-loadbalancer/50dc6c495c0c9188", + "backend": { + "port": "80", + "http": { + "response": { + "status_code": 200 + } + }, + "ip": "10.0.0.1" + }, + "target_group": { + "arn": "arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067" + }, + "backend_processing_time": { + "sec": 0.001 + }, + "action_executed": [ + "forward", + "redirect" + ] + } + }, + "user_agent": { + "name": "curl", + "original": "curl/7.46.0", + "device": { + "name": "Other" + }, + "version": "7.46.0" + } +} +``` + ## Metrics An example event for `elb` looks as following: @@ -166,7 +289,7 @@ An example event for `elb` looks as following: "period": 60000 }, "event": { - "dataset": "aws.elb", + "dataset": "aws.elb_metrics", "module": "aws", "duration": 15044430616 }, @@ -243,9 +366,10 @@ An example event for `elb` looks as following: | aws.networkelb.metrics.UnHealthyHostCount.max | The number of targets that are considered unhealthy. | long | | aws.s3.bucket.name | Name of a S3 bucket. | keyword | | aws.tags.\* | Tag key value pairs from aws resources. | object | +| cloud | Fields related to the cloud or infrastructure the events are coming from. | group | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | | cloud.instance.id | Instance ID of the host machine. | keyword | | cloud.instance.name | Instance name of the host machine. | keyword | @@ -260,7 +384,11 @@ An example event for `elb` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. | keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| error | These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. | group | +| error.message | Error message. | match_only_text | +| event.dataset | Event dataset | constant_keyword | +| event.module | Event module | constant_keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -277,5 +405,5 @@ An example event for `elb` looks as following: | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| service.type | Service type | keyword | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | diff --git a/test/packages/aws/docs/lambda.md b/test/packages/aws/docs/lambda.md index 8bbb73075d..202757eb37 100644 --- a/test/packages/aws/docs/lambda.md +++ b/test/packages/aws/docs/lambda.md @@ -15,7 +15,7 @@ An example event for `lambda` looks as following: "version": "8.0.0" }, "event": { - "dataset": "aws.dynamodb", + "dataset": "aws.lambda", "module": "aws", "duration": 10266182336 }, @@ -91,16 +91,17 @@ An example event for `lambda` looks as following: | aws.lambda.metrics.UnreservedConcurrentExecutions.avg | For an AWS Region, the number of events that are being processed by functions that don't have reserved concurrency. | double | | aws.s3.bucket.name | Name of a S3 bucket. | keyword | | aws.tags.\* | Tag key value pairs from aws resources. | object | +| cloud | Fields related to the cloud or infrastructure the events are coming from. | group | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | | cloud.instance.id | Instance ID of the host machine. | keyword | | cloud.instance.name | Instance name of the host machine. | keyword | | cloud.machine.type | Machine type of the host machine. | keyword | | cloud.project.id | Name of the project in Google Cloud. | keyword | | cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | +| cloud.region | Region in which this host, resource, or service is located. | keyword | | container.id | Unique container id. | keyword | | container.image.name | Name of the image the container was built on. | keyword | | container.labels | Image labels. | object | @@ -108,7 +109,11 @@ An example event for `lambda` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. | keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| error | These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. | group | +| error.message | Error message. | match_only_text | +| event.dataset | Event dataset | constant_keyword | +| event.module | Event module | constant_keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -125,5 +130,5 @@ An example event for `lambda` looks as following: | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| service.type | Service type | keyword | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | diff --git a/test/packages/aws/docs/natgateway.md b/test/packages/aws/docs/natgateway.md index b88d7108fc..3cba8c6434 100644 --- a/test/packages/aws/docs/natgateway.md +++ b/test/packages/aws/docs/natgateway.md @@ -116,16 +116,17 @@ An example event for `natgateway` looks as following: | aws.natgateway.metrics.PacketsOutToSource.sum | The number of packets sent through the NAT gateway to the clients in your VPC. | long | | aws.s3.bucket.name | Name of a S3 bucket. | keyword | | aws.tags.\* | Tag key value pairs from aws resources. | object | +| cloud | Fields related to the cloud or infrastructure the events are coming from. | group | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | | cloud.instance.id | Instance ID of the host machine. | keyword | | cloud.instance.name | Instance name of the host machine. | keyword | | cloud.machine.type | Machine type of the host machine. | keyword | | cloud.project.id | Name of the project in Google Cloud. | keyword | | cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | +| cloud.region | Region in which this host, resource, or service is located. | keyword | | container.id | Unique container id. | keyword | | container.image.name | Name of the image the container was built on. | keyword | | container.labels | Image labels. | object | @@ -133,7 +134,11 @@ An example event for `natgateway` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. | keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| error | These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. | group | +| error.message | Error message. | match_only_text | +| event.dataset | Event dataset | constant_keyword | +| event.module | Event module | constant_keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -150,4 +155,4 @@ An example event for `natgateway` looks as following: | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| service.type | Service type | keyword | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | diff --git a/test/packages/aws/docs/rds.md b/test/packages/aws/docs/rds.md index 4ac41d328d..27acd89791 100644 --- a/test/packages/aws/docs/rds.md +++ b/test/packages/aws/docs/rds.md @@ -102,6 +102,7 @@ An example event for `rds` looks as following: |---|---|---| | @timestamp | Event timestamp. | date | | aws.\*.metrics.\*.\* | Metrics that returned from Cloudwatch API query. | object | +| aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword | | aws.dimensions.\* | Metric dimensions. | object | | aws.dimensions.DBClusterIdentifier | This dimension filters the data that you request for a specific Amazon Aurora DB cluster. | keyword | | aws.dimensions.DBClusterIdentifier,Role | This dimension filters the data that you request for a specific Aurora DB cluster, aggregating the metric by instance role (WRITER/READER). | keyword | @@ -186,6 +187,7 @@ An example event for `rds` looks as following: | aws.rds.write_io.ops_per_sec | The average number of disk write I/O operations per second. | float | | aws.s3.bucket.name | Name of a S3 bucket. | keyword | | aws.tags.\* | Tag key value pairs from aws resources. | object | +| cloud | Fields related to the cloud or infrastructure the events are coming from. | group | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | | cloud.availability_zone | Availability zone in which this host is running. | keyword | @@ -203,7 +205,11 @@ An example event for `rds` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. | keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| error | These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. | group | +| error.message | Error message. | match_only_text | +| event.dataset | Event dataset | constant_keyword | +| event.module | Event module | constant_keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -220,4 +226,4 @@ An example event for `rds` looks as following: | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| service.type | Service type | keyword | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | diff --git a/test/packages/aws/docs/s3.md b/test/packages/aws/docs/s3.md index eb4ad6c434..c540a662db 100644 --- a/test/packages/aws/docs/s3.md +++ b/test/packages/aws/docs/s3.md @@ -38,9 +38,9 @@ for sending server access logs to S3 bucket. | aws.s3access.turn_around_time | The number of milliseconds that Amazon S3 spent processing your request. | long | | aws.s3access.user_agent | The value of the HTTP User-Agent header. | keyword | | aws.s3access.version_id | The version ID in the request, or "-" if the operation does not take a versionId parameter. | keyword | -| client.address | Some event client addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the .address field. | keyword | -| client.ip | IP address of the client. | ip | -| client.user.id | Unique identifiers of the user. | keyword | +| client.address | Some event client addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | +| client.ip | IP address of the client (IPv4 or IPv6). | ip | +| client.user.id | Unique identifier of the user. | keyword | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | @@ -57,12 +57,16 @@ for sending server access logs to S3 bucket. | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| event.action | The action captured by the event. | keyword | -| event.code | Identification code for this event, if one exists. | keyword | -| event.duration | Duration of the event in nanoseconds. | long | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| error.message | Error message. | match_only_text | +| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | +| event.code | Identification code for this event, if one exists. Some event sources use event codes to identify messages unambiguously, regardless of message language or wording adjustments over time. An example of this is the Windows Event ID. | keyword | +| event.dataset | Event dataset | constant_keyword | +| event.duration | Duration of the event in nanoseconds. If event.start and event.end are known this value should be the difference between the end and start time. | long | | event.id | Unique ID to describe the event. | keyword | -| event.kind | Event kind (e.g. event, alert, metric, state, pipeline_error, signal) | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. | keyword | +| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data coming in at a regular interval or not. | keyword | +| event.module | Event module | constant_keyword | +| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | | geo.city_name | City name. | keyword | | geo.continent_name | Name of the continent. | keyword | | geo.country_iso_code | Country ISO code. | keyword | @@ -86,19 +90,23 @@ for sending server access logs to S3 bucket. | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| http.request.method | HTTP request method. | keyword | +| http.request.method | HTTP request method. Prior to ECS 1.6.0 the following guidance was provided: "The field value must be normalized to lowercase for querying." As of ECS 1.6.0, the guidance is deprecated because the original case of the method may be useful in anomaly detection. Original case will be mandated in ECS 2.0.0 | keyword | | http.request.referrer | Referrer for this HTTP request. | keyword | | http.response.body.bytes | Size in bytes of the response body. | long | | http.response.status_code | HTTP response status code. | long | | http.version | HTTP version. | keyword | | related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names seen on your event. | keyword | +| related.user | All the user names or other user identifiers seen on the event. | keyword | +| tags | List of keywords used to tag each event. | keyword | | tls.cipher | String indicating the cipher used during the current connection. | keyword | | tls.version | Numeric part of the version parsed from the original string. | keyword | | tls.version_protocol | Normalized lowercase protocol name parsed from original string. | keyword | -| url.original | Unmodified original url as seen in the event source. | keyword | -| url.path | Path of the request, such as "/search". | keyword | -| url.query | The query field describes the query string of the request, such as "q=elasticsearch". | keyword | +| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | +| url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | +| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | +| url.path | Path of the request, such as "/search". | wildcard | +| url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | +| url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | | user_agent.device.name | Name of the device. | keyword | | user_agent.name | Name of the user agent. | keyword | | user_agent.original | Unparsed user_agent string. | keyword | @@ -108,6 +116,124 @@ for sending server access logs to S3 bucket. | user_agent.version | Version of the user agent. | keyword | +An example event for `s3access` looks as following: + +```json +{ + "@timestamp": "2021-11-26T14:44:27.652Z", + "data_stream": { + "namespace": "default", + "type": "logs", + "dataset": "aws.s3access" + }, + "url": { + "path": "/test-s3-ks/", + "original": "/test-s3-ks/?location\u0026aws-account=627959692251", + "query": "location\u0026aws-account=627959692251" + }, + "tags": [ + "preserve_original_event" + ], + "geo": { + "continent_name": "North America", + "region_iso_code": "US-VA", + "city_name": "Ashburn", + "country_iso_code": "US", + "country_name": "United States", + "region_name": "Virginia", + "location": { + "lon": -77.4728, + "lat": 39.0481 + } + }, + "cloud": { + "provider": "aws" + }, + "ecs": { + "version": "1.12.0" + }, + "related": { + "user": [ + "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2" + ], + "ip": [ + "72.21.217.31" + ] + }, + "http": { + "request": { + "method": "GET" + }, + "version": "1.1", + "response": { + "body": { + "bytes": 142 + }, + "status_code": 200 + } + }, + "client": { + "user": { + "id": "arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9" + }, + "address": "72.21.217.31", + "ip": "72.21.217.31" + }, + "tls": { + "cipher": "ECDHE-RSA-AES128-SHA", + "version": "1.2", + "version_protocol": "tls" + }, + "event": { + "duration": 17000000, + "ingested": "2021-07-19T21:47:05.259665700Z", + "original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:41 +0000] 72.21.217.31 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 44EE8651683CB4DA REST.GET.LOCATION - \"GET /test-s3-ks/?location\u0026aws-account=627959692251 HTTP/1.1\" 200 - 142 - 17 - \"-\" \"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation\" - BsCfJedfuSnds2QFoxi+E/O7M6OEWzJnw4dUaes/2hyA363sONRJKzB7EOY+Bt9DTHYUn+HoHxI= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2", + "kind": "event", + "action": "REST.GET.LOCATION", + "id": "44EE8651683CB4DA", + "category": "web", + "type": [ + "access" + ], + "outcome": "success" + }, + "aws": { + "s3access": { + "requester": "arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9", + "tls_version": "TLSv1.2", + "signature_version": "SigV4", + "bytes_sent": 142, + "authentication_type": "AuthHeader", + "request_uri": "GET /test-s3-ks/?location\u0026aws-account=627959692251 HTTP/1.1", + "host_id": "BsCfJedfuSnds2QFoxi+E/O7M6OEWzJnw4dUaes/2hyA363sONRJKzB7EOY+Bt9DTHYUn+HoHxI=", + "host_header": "s3.ap-southeast-1.amazonaws.com", + "bucket": "test-s3-ks", + "remote_ip": "72.21.217.31", + "cipher_suite": "ECDHE-RSA-AES128-SHA", + "http_status": 200, + "total_time": 17, + "bucket_owner": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2", + "operation": "REST.GET.LOCATION", + "request_id": "44EE8651683CB4DA", + "user_agent": "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation" + } + }, + "user_agent": { + "name": "aws-sdk-java", + "original": "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation", + "os": { + "name": "Linux", + "version": "4.9.137", + "full": "Linux 4.9.137" + }, + "device": { + "name": "Other" + }, + "version": "1.11.590" + } +} +``` + ## Metrics ### s3_daily_storage @@ -179,9 +305,10 @@ An example event for `s3_daily_storage` looks as following: | aws.s3_daily_storage.bucket.size.bytes | The amount of data in bytes stored in a bucket. | long | | aws.s3_daily_storage.number_of_objects | The total number of objects stored in a bucket for all storage classes. | long | | aws.tags.\* | Tag key value pairs from aws resources. | object | +| cloud | Fields related to the cloud or infrastructure the events are coming from. | group | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | | cloud.instance.id | Instance ID of the host machine. | keyword | | cloud.instance.name | Instance name of the host machine. | keyword | @@ -196,7 +323,11 @@ An example event for `s3_daily_storage` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. | keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| error | These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. | group | +| error.message | Error message. | match_only_text | +| event.dataset | Event dataset | constant_keyword | +| event.module | Event module | constant_keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -213,7 +344,7 @@ An example event for `s3_daily_storage` looks as following: | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| service.type | Service type | keyword | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | ### s3_request @@ -312,16 +443,17 @@ An example event for `s3_request` looks as following: | aws.s3_request.requests.total | The total number of HTTP requests made to an Amazon S3 bucket, regardless of type. | long | | aws.s3_request.uploaded.bytes | The number bytes uploaded that contain a request body, made to an Amazon S3 bucket. | long | | aws.tags.\* | Tag key value pairs from aws resources. | object | +| cloud | Fields related to the cloud or infrastructure the events are coming from. | group | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | | cloud.instance.id | Instance ID of the host machine. | keyword | | cloud.instance.name | Instance name of the host machine. | keyword | | cloud.machine.type | Machine type of the host machine. | keyword | | cloud.project.id | Name of the project in Google Cloud. | keyword | | cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | +| cloud.region | Region in which this host, resource, or service is located. | keyword | | container.id | Unique container id. | keyword | | container.image.name | Name of the image the container was built on. | keyword | | container.labels | Image labels. | object | @@ -329,7 +461,11 @@ An example event for `s3_request` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. | keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| error | These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. | group | +| error.message | Error message. | match_only_text | +| event.dataset | Event dataset | constant_keyword | +| event.module | Event module | constant_keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -346,5 +482,5 @@ An example event for `s3_request` looks as following: | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| service.type | Service type | keyword | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | diff --git a/test/packages/aws/docs/s3_storage_lens.md b/test/packages/aws/docs/s3_storage_lens.md new file mode 100644 index 0000000000..58b66cb8ab --- /dev/null +++ b/test/packages/aws/docs/s3_storage_lens.md @@ -0,0 +1,226 @@ +# s3 storage lens + +## Metrics + +An example event for `s3_storage_lens` looks as following: + +```json +{ + "@timestamp": "2021-11-07T20:38:00.000Z", + "ecs": { + "version": "1.11.0" + }, + "data_stream": { + "namespace": "default", + "type": "metrics", + "dataset": "aws.s3_storage_lens" + }, + "service": { + "type": "aws" + }, + "cloud": { + "provider": "aws", + "region": "us-east-1", + "account": { + "name": "elastic-beats", + "id": "428152502467" + } + }, + "metricset": { + "period": 86400000, + "name": "cloudwatch" + }, + "event": { + "duration": 22973251900, + "agent_id_status": "verified", + "ingested": "2021-11-08T20:38:37Z", + "module": "aws", + "dataset": "aws.s3_storage_lens" + }, + "aws": { + "s3_storage_lens": { + "metrics": { + "NonCurrentVersionStorageBytes": { + "avg": 0 + }, + "DeleteMarkerObjectCount": { + "avg": 0 + }, + "GetRequests": { + "avg": 0 + }, + "SelectReturnedBytes": { + "avg": 0 + }, + "ObjectCount": { + "avg": 164195 + }, + "HeadRequests": { + "avg": 0 + }, + "ListRequests": { + "avg": 0 + }, + "DeleteRequests": { + "avg": 0 + }, + "SelectRequests": { + "avg": 0 + }, + "5xxErrors": { + "avg": 0 + }, + "BytesDownloaded": { + "avg": 0 + }, + "BytesUploaded": { + "avg": 82537 + }, + "CurrentVersionStorageBytes": { + "avg": 154238334 + }, + "StorageBytes": { + "avg": 154238334 + }, + "ObjectLockEnabledStorageBytes": { + "avg": 0 + }, + "4xxErrors": { + "avg": 0 + }, + "PutRequests": { + "avg": 145 + }, + "ObjectLockEnabledObjectCount": { + "avg": 0 + }, + "EncryptedObjectCount": { + "avg": 164191 + }, + "CurrentVersionObjectCount": { + "avg": 164195 + }, + "IncompleteMultipartUploadObjectCount": { + "avg": 0 + }, + "ReplicatedObjectCount": { + "avg": 0 + }, + "AllRequests": { + "avg": 145 + }, + "PostRequests": { + "avg": 0 + }, + "IncompleteMultipartUploadStorageBytes": { + "avg": 0 + }, + "NonCurrentVersionObjectCount": { + "avg": 0 + }, + "ReplicatedStorageBytes": { + "avg": 0 + }, + "EncryptedStorageBytes": { + "avg": 154237917 + }, + "SelectScannedBytes": { + "avg": 0 + } + } + }, + "cloudwatch": { + "namespace": "AWS/S3/Storage-Lens" + }, + "dimensions": { + "metrics_version": "1.0", + "storage_class": "STANDARD", + "aws_region": "eu-central-1", + "bucket_name": "filebeat-aws-elb-test", + "aws_account_number": "428152502467", + "configuration_id": "default-account-dashboard", + "record_type": "BUCKET" + } + } +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| aws.\*.metrics.\*.\* | Metrics that returned from Cloudwatch API query. | object | +| aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword | +| aws.dimensions.\* | Metric dimensions. | object | +| aws.s3.bucket.name | Name of a S3 bucket. | keyword | +| aws.s3_storage_lens.metrics.4xxErrors.avg | The total 4xx errors in scope. | long | +| aws.s3_storage_lens.metrics.5xxErrors.avg | The total 5xx errors in scope. | long | +| aws.s3_storage_lens.metrics.AllRequests.avg | The total number of requests made. | long | +| aws.s3_storage_lens.metrics.BytesDownloaded.avg | The number of bytes in scope that were downloaded. | long | +| aws.s3_storage_lens.metrics.BytesUploaded.avg | The number of bytes uploaded. | long | +| aws.s3_storage_lens.metrics.CurrentVersionObjectCount.avg | The number of objects that are a current version. | long | +| aws.s3_storage_lens.metrics.CurrentVersionStorageBytes.avg | The number of bytes that are a current version. | long | +| aws.s3_storage_lens.metrics.DeleteMarkerObjectCount.avg | The total number of objects with a delete marker. | long | +| aws.s3_storage_lens.metrics.DeleteRequests.avg | The total number of delete requests made. | long | +| aws.s3_storage_lens.metrics.EncryptedObjectCount.avg | The total object counts that are encrypted using Amazon S3 server-side encryption. | long | +| aws.s3_storage_lens.metrics.EncryptedStorageBytes.avg | The total number of encrypted bytes using Amazon S3 server-side encryption. | long | +| aws.s3_storage_lens.metrics.GetRequests.avg | The total number of GET requests made. | long | +| aws.s3_storage_lens.metrics.HeadRequests.avg | The total number of head requests made. | long | +| aws.s3_storage_lens.metrics.IncompleteMultipartUploadObjectCount.avg | The number of objects in scope that are incomplete multipart uploads. | long | +| aws.s3_storage_lens.metrics.IncompleteMultipartUploadStorageBytes.avg | The total bytes in scope with incomplete multipart uploads. | long | +| aws.s3_storage_lens.metrics.ListRequests.avg | The total number of list requests made. | long | +| aws.s3_storage_lens.metrics.NonCurrentVersionObjectCount.avg | The count of the noncurrent version objects. | long | +| aws.s3_storage_lens.metrics.NonCurrentVersionStorageBytes.avg | The number of noncurrent versioned bytes. | long | +| aws.s3_storage_lens.metrics.ObjectCount.avg | The total object count. | long | +| aws.s3_storage_lens.metrics.ObjectLockEnabledObjectCount.avg | The total number of objects in scope that have Object Lock enabled. | long | +| aws.s3_storage_lens.metrics.ObjectLockEnabledStorageBytes.avg | The total number of bytes in scope that have Object Lock enabled. | long | +| aws.s3_storage_lens.metrics.PostRequests.avg | The total number of post requests made. | long | +| aws.s3_storage_lens.metrics.PutRequests.avg | The total number of PUT requests made. | long | +| aws.s3_storage_lens.metrics.ReplicatedObjectCount.avg | The count of replicated objects. | long | +| aws.s3_storage_lens.metrics.ReplicatedStorageBytes.avg | The total number of bytes in scope that are replicated. | long | +| aws.s3_storage_lens.metrics.SelectRequests.avg | The total number of select requests. | long | +| aws.s3_storage_lens.metrics.SelectReturnedBytes.avg | The number of select bytes returned. | long | +| aws.s3_storage_lens.metrics.SelectScannedBytes.avg | The number of select bytes scanned. | long | +| aws.s3_storage_lens.metrics.StorageBytes.avg | The total storage in bytes | long | +| aws.tags.\* | Tag key value pairs from aws resources. | object | +| cloud | Fields related to the cloud or infrastructure the events are coming from. | group | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | +| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| error | These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. | group | +| error.message | Error message. | match_only_text | +| event.dataset | Event dataset | constant_keyword | +| event.module | Event module | constant_keyword | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | diff --git a/test/packages/aws/docs/sns.md b/test/packages/aws/docs/sns.md index 2c2905fd09..1dc2ebcfd8 100644 --- a/test/packages/aws/docs/sns.md +++ b/test/packages/aws/docs/sns.md @@ -91,16 +91,17 @@ An example event for `sns` looks as following: | aws.sns.metrics.SMSMonthToDateSpentUSD.sum | The charges you have accrued since the start of the current calendar month for sending SMS messages. | long | | aws.sns.metrics.SMSSuccessRate.avg | The rate of successful SMS message deliveries. | double | | aws.tags.\* | Tag key value pairs from aws resources. | object | +| cloud | Fields related to the cloud or infrastructure the events are coming from. | group | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | | cloud.instance.id | Instance ID of the host machine. | keyword | | cloud.instance.name | Instance name of the host machine. | keyword | | cloud.machine.type | Machine type of the host machine. | keyword | | cloud.project.id | Name of the project in Google Cloud. | keyword | | cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | +| cloud.region | Region in which this host, resource, or service is located. | keyword | | container.id | Unique container id. | keyword | | container.image.name | Name of the image the container was built on. | keyword | | container.labels | Image labels. | object | @@ -108,7 +109,11 @@ An example event for `sns` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. | keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| error | These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. | group | +| error.message | Error message. | match_only_text | +| event.dataset | Event dataset | constant_keyword | +| event.module | Event module | constant_keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -125,4 +130,4 @@ An example event for `sns` looks as following: | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| service.type | Service type | keyword | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | diff --git a/test/packages/aws/docs/sqs.md b/test/packages/aws/docs/sqs.md index 2a6dc4ca79..72d9b68bf2 100644 --- a/test/packages/aws/docs/sqs.md +++ b/test/packages/aws/docs/sqs.md @@ -66,6 +66,7 @@ An example event for `sqs` looks as following: |---|---|---| | @timestamp | Event timestamp. | date | | aws.\*.metrics.\*.\* | Metrics that returned from Cloudwatch API query. | object | +| aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword | | aws.dimensions.\* | Metric dimensions. | object | | aws.dimensions.QueueName | SQS queue name | keyword | | aws.s3.bucket.name | Name of a S3 bucket. | keyword | @@ -80,9 +81,10 @@ An example event for `sqs` looks as following: | aws.sqs.queue.name | SQS queue name | keyword | | aws.sqs.sent_message_size.bytes | The size of messages added to a queue. | long | | aws.tags.\* | Tag key value pairs from aws resources. | object | +| cloud | Fields related to the cloud or infrastructure the events are coming from. | group | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | | cloud.instance.id | Instance ID of the host machine. | keyword | | cloud.instance.name | Instance name of the host machine. | keyword | @@ -97,7 +99,11 @@ An example event for `sqs` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. | keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| error | These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. | group | +| error.message | Error message. | match_only_text | +| event.dataset | Event dataset | constant_keyword | +| event.module | Event module | constant_keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -114,4 +120,4 @@ An example event for `sqs` looks as following: | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| service.type | Service type | keyword | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | diff --git a/test/packages/aws/docs/transitgateway.md b/test/packages/aws/docs/transitgateway.md index f8c99e8b6f..ed4cc85cfd 100644 --- a/test/packages/aws/docs/transitgateway.md +++ b/test/packages/aws/docs/transitgateway.md @@ -88,16 +88,17 @@ An example event for `transitgateway` looks as following: | aws.transitgateway.metrics.PacketDropCountNoRoute.sum | The number of packets dropped because they did not match a route. | long | | aws.transitgateway.metrics.PacketsIn.sum | The number of packets received by the transit gateway. | long | | aws.transitgateway.metrics.PacketsOut.sum | The number of packets sent by the transit gateway. | long | +| cloud | Fields related to the cloud or infrastructure the events are coming from. | group | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | | cloud.instance.id | Instance ID of the host machine. | keyword | | cloud.instance.name | Instance name of the host machine. | keyword | | cloud.machine.type | Machine type of the host machine. | keyword | | cloud.project.id | Name of the project in Google Cloud. | keyword | | cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | +| cloud.region | Region in which this host, resource, or service is located. | keyword | | container.id | Unique container id. | keyword | | container.image.name | Name of the image the container was built on. | keyword | | container.labels | Image labels. | object | @@ -105,7 +106,11 @@ An example event for `transitgateway` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. | keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| error | These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. | group | +| error.message | Error message. | match_only_text | +| event.dataset | Event dataset | constant_keyword | +| event.module | Event module | constant_keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -122,5 +127,5 @@ An example event for `transitgateway` looks as following: | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| service.type | Service type | keyword | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | diff --git a/test/packages/aws/docs/usage.md b/test/packages/aws/docs/usage.md index 4a6d7009fc..2dfc1b966a 100644 --- a/test/packages/aws/docs/usage.md +++ b/test/packages/aws/docs/usage.md @@ -74,6 +74,7 @@ An example event for `usage` looks as following: | aws.tags.\* | Tag key value pairs from aws resources. | object | | aws.usage.metrics.CallCount.sum | The number of specified API operations performed in your account. | long | | aws.usage.metrics.ResourceCount.sum | The number of the specified resources running in your account. The resources are defined by the dimensions associated with the metric. | long | +| cloud | Fields related to the cloud or infrastructure the events are coming from. | group | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | | cloud.availability_zone | Availability zone in which this host is running. | keyword | @@ -83,7 +84,7 @@ An example event for `usage` looks as following: | cloud.machine.type | Machine type of the host machine. | keyword | | cloud.project.id | Name of the project in Google Cloud. | keyword | | cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | +| cloud.region | Region in which this host, resource, or service is located. | keyword | | container.id | Unique container id. | keyword | | container.image.name | Name of the image the container was built on. | keyword | | container.labels | Image labels. | object | @@ -91,7 +92,11 @@ An example event for `usage` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. | keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| error | These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. | group | +| error.message | Error message. | match_only_text | +| event.dataset | Event dataset | constant_keyword | +| event.module | Event module | constant_keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -108,5 +113,5 @@ An example event for `usage` looks as following: | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| service.type | Service type | keyword | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | diff --git a/test/packages/aws/docs/vpcflow.md b/test/packages/aws/docs/vpcflow.md index c2eadee598..09e03fc0ae 100644 --- a/test/packages/aws/docs/vpcflow.md +++ b/test/packages/aws/docs/vpcflow.md @@ -16,17 +16,18 @@ | aws.vpcflow.pkt_srcaddr | The packet-level (original) source IP address of the traffic. | ip | | aws.vpcflow.subnet_id | The ID of the subnet that contains the network interface for which the traffic is recorded. | keyword | | aws.vpcflow.tcp_flags | The bitmask value for the following TCP flags: 2=SYN,18=SYN-ACK,1=FIN,4=RST | keyword | +| aws.vpcflow.tcp_flags_array | List of TCP flags: 'fin, syn, rst, psh, ack, urg' | keyword | | aws.vpcflow.type | The type of traffic: IPv4, IPv6, or EFA. | keyword | | aws.vpcflow.version | The VPC Flow Logs version. If you use the default format, the version is 2. If you specify a custom format, the version is 3. | keyword | | aws.vpcflow.vpc_id | The ID of the VPC that contains the network interface for which the traffic is recorded. | keyword | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. | keyword | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | | cloud.instance.id | Instance ID of the host machine. | keyword | | cloud.instance.name | Instance name of the host machine. | keyword | | cloud.machine.type | Machine type of the host machine. | keyword | | cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | | cloud.region | Region in which this host is running. | keyword | | container.id | Unique container id. | keyword | | container.image.name | Name of the image the container was built on. | keyword | @@ -35,21 +36,29 @@ | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.address | Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the .address field. | keyword | +| destination.address | Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | | destination.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | | destination.as.organization.name | Organization name. | keyword | +| destination.geo.city_name | City name. | keyword | | destination.geo.continent_name | Name of the continent. | keyword | | destination.geo.country_iso_code | Country ISO code. | keyword | +| destination.geo.country_name | Country name. | keyword | | destination.geo.location | Longitude and latitude. | geo_point | -| destination.ip | IP address of the destination. | ip | +| destination.geo.region_iso_code | Region ISO code. | keyword | +| destination.geo.region_name | Region name. | keyword | +| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | | destination.port | Port of the destination. | long | -| event.category | Event category (e.g. database) | keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| error.message | Error message. | match_only_text | +| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | +| event.dataset | Event dataset | constant_keyword | | event.end | event.end contains the date when the event ended or when the activity was last observed. | date | -| event.kind | Event kind (e.g. event, alert, metric, state, pipeline_error, signal) | keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity. | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. | keyword | +| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data coming in at a regular interval or not. | keyword | +| event.module | Event module | constant_keyword | +| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | +| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | | event.start | event.start contains the date when the event started or when the activity was first observed. | date | -| event.type | Event severity (e.g. info, error) | keyword | +| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -66,24 +75,96 @@ | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| network.bytes | Total bytes transferred in both directions. | long | -| network.community_id | A hash of source and destination IPs and ports, as well as the protocol used in a communication. This is a tool-agnostic standard to identify flows. | keyword | +| network.bytes | Total bytes transferred in both directions. If `source.bytes` and `destination.bytes` are known, `network.bytes` is their sum. | long | +| network.community_id | A hash of source and destination IPs and ports, as well as the protocol used in a communication. This is a tool-agnostic standard to identify flows. Learn more at https://github.com/corelight/community-id-spec. | keyword | | network.iana_number | IANA Protocol Number (https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml). Standardized list of protocols. This aligns well with NetFlow and sFlow related logs which use the IANA Protocol Number. | keyword | -| network.packets | Total packets transferred in both directions. | long | -| network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) | keyword | -| network.type | In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc | keyword | +| network.packets | Total packets transferred in both directions. If `source.packets` and `destination.packets` are known, `network.packets` is their sum. | long | +| network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". | keyword | +| network.type | In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". | keyword | | related.ip | All of the IPs seen on your event. | ip | -| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the .address field. | keyword | +| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | | source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | | source.as.organization.name | Organization name. | keyword | | source.bytes | Bytes sent from the source to the destination. | long | | source.geo.city_name | City name. | keyword | | source.geo.continent_name | Name of the continent. | keyword | | source.geo.country_iso_code | Country ISO code. | keyword | +| source.geo.country_name | Country name. | keyword | | source.geo.location | Longitude and latitude. | geo_point | | source.geo.region_iso_code | Region ISO code. | keyword | | source.geo.region_name | Region name. | keyword | | source.ip | IP address of the source (IPv4 or IPv6). | ip | | source.packets | Packets sent from the source to the destination. | long | | source.port | Port of the source. | long | +| tags | List of keywords used to tag each event. | keyword | + +An example event for `vpcflow` looks as following: + +```json +{ + "data_stream": { + "namespace": "default", + "type": "logs", + "dataset": "aws.vpcflow" + }, + "destination": { + "port": 22, + "address": "2001:db8:1234:a102:3304:8879:34cf:4071", + "ip": "2001:db8:1234:a102:3304:8879:34cf:4071" + }, + "source": { + "address": "2001:db8:1234:a100:8d6e:3477:df66:f105", + "port": 34892, + "bytes": 8855, + "packets": 54, + "ip": "2001:db8:1234:a100:8d6e:3477:df66:f105" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "community_id": "1:hXZclvxUJScaVf0xMIJR6yW6tBQ=", + "transport": "tcp", + "type": "ipv6", + "bytes": 8855, + "iana_number": "6", + "packets": 54 + }, + "cloud": { + "provider": "aws", + "account": { + "id": "123456789010" + } + }, + "@timestamp": "2016-10-31T11:37:00.000Z", + "ecs": { + "version": "1.12.0" + }, + "related": { + "ip": [ + "2001:db8:1234:a100:8d6e:3477:df66:f105", + "2001:db8:1234:a102:3304:8879:34cf:4071" + ] + }, + "event": { + "ingested": "2021-09-28T19:10:43.075027100Z", + "original": "2 123456789010 eni-1235b8ca123456789 2001:db8:1234:a100:8d6e:3477:df66:f105 2001:db8:1234:a102:3304:8879:34cf:4071 34892 22 6 54 8855 1477913708 1477913820 ACCEPT OK", + "kind": "event", + "start": "2016-10-31T11:35:08.000Z", + "end": "2016-10-31T11:37:00.000Z", + "type": "flow", + "category": "network_traffic", + "outcome": "allow" + }, + "aws": { + "vpcflow": { + "action": "ACCEPT", + "account_id": "123456789010", + "log_status": "OK", + "interface_id": "eni-1235b8ca123456789", + "version": "2" + } + } +} +``` \ No newline at end of file diff --git a/test/packages/aws/docs/vpn.md b/test/packages/aws/docs/vpn.md index 70e6f1aefc..ea956e4b42 100644 --- a/test/packages/aws/docs/vpn.md +++ b/test/packages/aws/docs/vpn.md @@ -73,6 +73,7 @@ An example event for `vpn` looks as following: | aws.vpn.metrics.TunnelDataIn.sum | The bytes received through the VPN tunnel. | double | | aws.vpn.metrics.TunnelDataOut.sum | The bytes sent through the VPN tunnel. | double | | aws.vpn.metrics.TunnelState.avg | The state of the tunnel. For static VPNs, 0 indicates DOWN and 1 indicates UP. For BGP VPNs, 1 indicates ESTABLISHED and 0 is used for all other states. | double | +| cloud | Fields related to the cloud or infrastructure the events are coming from. | group | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | | cloud.availability_zone | Availability zone in which this host is running. | keyword | @@ -82,7 +83,7 @@ An example event for `vpn` looks as following: | cloud.machine.type | Machine type of the host machine. | keyword | | cloud.project.id | Name of the project in Google Cloud. | keyword | | cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | +| cloud.region | Region in which this host, resource, or service is located. | keyword | | container.id | Unique container id. | keyword | | container.image.name | Name of the image the container was built on. | keyword | | container.labels | Image labels. | object | @@ -90,7 +91,11 @@ An example event for `vpn` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. | keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| error | These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. | group | +| error.message | Error message. | match_only_text | +| event.dataset | Event dataset | constant_keyword | +| event.module | Event module | constant_keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -107,5 +112,5 @@ An example event for `vpn` looks as following: | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| service.type | Service type | keyword | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | diff --git a/test/packages/aws/docs/waf.md b/test/packages/aws/docs/waf.md new file mode 100644 index 0000000000..3cd35ac5a6 --- /dev/null +++ b/test/packages/aws/docs/waf.md @@ -0,0 +1,182 @@ +# waf + +## Logs + +The `waf` dataset is specifically for WAF logs. Export logs from Kinesis Data Firehose to Amazon S3 bucket which has SQS notification setup already. + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| aws.waf.arn | AWS ARN of ACL | keyword | +| aws.waf.id | ID of ACL | keyword | +| aws.waf.non_terminating_matching_rules | The list of non-terminating rules in the rule group that match the request. These are always COUNT rules (non-terminating rules that match) | nested | +| aws.waf.rate_based_rule_list | The list of rate-based rules that acted on the request. | nested | +| aws.waf.request.headers | List of request headers | flattened | +| aws.waf.rule_group_list | The list of rule groups that acted on this request. | nested | +| aws.waf.source.id | The source ID. This field shows the ID of the associated resource. | keyword | +| aws.waf.source.name | The source of the request. Possible values: CF for Amazon CloudFront, APIGW for Amazon API Gateway, ALB for Application Load Balancer, and APPSYNC for AWS AppSync. | keyword | +| aws.waf.terminating_rule_match_details | Detailed information about the terminating rule that matched the request. A terminating rule has an action that ends the inspection process against a web request. Possible actions for a terminating rule are ALLOW and BLOCK. This is only populated for SQL injection and cross-site scripting (XSS) match rule statements. As with all rule statements that inspect for more than one thing, AWS WAF applies the action on the first match and stops inspecting the web request. A web request with a terminating action could contain other threats, in addition to the one reported in the log. | nested | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| error.message | Error message. | match_only_text | +| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | +| event.dataset | Event dataset | constant_keyword | +| event.id | Unique ID to describe the event. | keyword | +| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data coming in at a regular interval or not. | keyword | +| event.module | Event module | constant_keyword | +| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| http.request.id | A unique identifier for each HTTP request to correlate logs between clients and servers in transactions. The id may be contained in a non-standard HTTP header, such as `X-Request-ID` or `X-Correlation-ID`. | keyword | +| http.request.method | HTTP request method. Prior to ECS 1.6.0 the following guidance was provided: "The field value must be normalized to lowercase for querying." As of ECS 1.6.0, the guidance is deprecated because the original case of the method may be useful in anomaly detection. Original case will be mandated in ECS 2.0.0 | keyword | +| http.version | HTTP version. | keyword | +| network.protocol | L7 Network protocol name. ex. http, lumberjack, transport protocol. The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". | keyword | +| network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". | keyword | +| related.ip | All of the IPs seen on your event. | ip | +| rule.id | A rule ID that is unique within the scope of an agent, observer, or other entity using the rule for detection of this event. | keyword | +| rule.ruleset | Name of the ruleset, policy, group, or parent category in which the rule used to generate this event is a member. | keyword | +| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | +| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | +| source.as.organization.name | Organization name. | keyword | +| source.geo.city_name | City name. | keyword | +| source.geo.continent_name | Name of the continent. | keyword | +| source.geo.country_iso_code | Country ISO code. | keyword | +| source.geo.country_name | Country name. | keyword | +| source.geo.location | Longitude and latitude. | geo_point | +| source.geo.region_iso_code | Region ISO code. | keyword | +| source.geo.region_name | Region name. | keyword | +| source.ip | IP address of the source (IPv4 or IPv6). | ip | +| tags | List of keywords used to tag each event. | keyword | +| url.path | Path of the request, such as "/search". | wildcard | +| url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | + + +An example event for `waf` looks as following: + +```json +{ + "@timestamp": "2021-11-25T14:25:25.000Z", + "data_stream": { + "namespace": "default", + "type": "logs", + "dataset": "aws.waf" + }, + "rule": { + "ruleset": "REGULAR", + "id": "STMTest_SQLi_XSS" + }, + "source": { + "geo": { + "continent_name": "Oceania", + "country_name": "Australia", + "location": { + "lon": 143.2104, + "lat": -33.494 + }, + "country_iso_code": "AU" + }, + "as": { + "number": 13335, + "organization": { + "name": "Cloudflare, Inc." + } + }, + "ip": "1.1.1.1" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "http", + "transport": "tcp" + }, + "cloud": { + "region": "ap-southeast-2", + "provider": "aws", + "service": { + "name": "wafv2" + }, + "account": { + "id": "12345" + } + }, + "ecs": { + "version": "1.12.0" + }, + "related": { + "ip": [ + "1.1.1.1" + ] + }, + "http": { + "request": { + "method": "POST", + "id": "null" + }, + "version": "1.1" + }, + "event": { + "action": "BLOCK", + "ingested": "2021-10-11T15:00:35.544818361Z", + "original": "{\"timestamp\":1576280412771,\"formatVersion\":1,\"webaclId\":\"arn:aws:wafv2:ap-southeast-2:12345:regional/webacl/test/111\",\"terminatingRuleId\":\"STMTest_SQLi_XSS\",\"terminatingRuleType\":\"REGULAR\",\"action\":\"BLOCK\",\"terminatingRuleMatchDetails\":[{\"conditionType\":\"SQL_INJECTION\",\"location\":\"UNKNOWN\",\"matchedData\":[\"10\",\"AND\",\"1\"]}],\"httpSourceName\":\"ALB\",\"httpSourceId\":\"alb\",\"ruleGroupList\":[],\"rateBasedRuleList\":[],\"nonTerminatingMatchingRules\":[],\"requestHeadersInserted\":null,\"responseCodeSent\":null,\"httpRequest\":{\"clientIp\":\"1.1.1.1\",\"country\":\"AU\",\"headers\":[],\"uri\":\"\",\"args\":\"\",\"httpVersion\":\"HTTP/1.1\",\"httpMethod\":\"POST\",\"requestId\":\"null\"},\"labels\":[{\"name\":\"value\"}]}", + "category": "web", + "type": [ + "access", + "denied" + ], + "kind": "event" + }, + "aws": { + "waf": { + "terminating_rule_match_details": [ + { + "conditionType": "SQL_INJECTION", + "location": "UNKNOWN", + "matchedData": [ + "10", + "AND", + "1" + ] + } + ], + "id": "regional/webacl/test/111", + "source": { + "name": "ALB", + "id": "alb" + }, + "arn": "arn:aws:wafv2:ap-southeast-2:12345:regional/webacl/test/111" + } + } +} +``` \ No newline at end of file diff --git a/test/packages/aws/img/logo_s3_storage_lens.svg b/test/packages/aws/img/logo_s3_storage_lens.svg new file mode 100644 index 0000000000..b1b8c0fb7e --- /dev/null +++ b/test/packages/aws/img/logo_s3_storage_lens.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/test/packages/aws/img/logo_waf.svg b/test/packages/aws/img/logo_waf.svg new file mode 100644 index 0000000000..b57e1ab481 --- /dev/null +++ b/test/packages/aws/img/logo_waf.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/test/packages/aws/img/metricbeat-aws-rds-overview.png b/test/packages/aws/img/metricbeat-aws-rds-overview.png index d44b021588..e7830e44d4 100644 Binary files a/test/packages/aws/img/metricbeat-aws-rds-overview.png and b/test/packages/aws/img/metricbeat-aws-rds-overview.png differ diff --git a/test/packages/aws/img/metricbeat-aws-s3-storage-lens-overview.png b/test/packages/aws/img/metricbeat-aws-s3-storage-lens-overview.png new file mode 100644 index 0000000000..5b0996416f Binary files /dev/null and b/test/packages/aws/img/metricbeat-aws-s3-storage-lens-overview.png differ diff --git a/test/packages/aws/kibana/dashboard/aws-3367c170-921f-11e9-aa19-159bf182e06f.json b/test/packages/aws/kibana/dashboard/aws-3367c170-921f-11e9-aa19-159bf182e06f.json index 3a169ee277..fa0bf6a54a 100644 --- a/test/packages/aws/kibana/dashboard/aws-3367c170-921f-11e9-aa19-159bf182e06f.json +++ b/test/packages/aws/kibana/dashboard/aws-3367c170-921f-11e9-aa19-159bf182e06f.json @@ -18,175 +18,863 @@ "panelsJSON": [ { "embeddableConfig": { - "title": "Database Connections" + "enhancements": {} }, "gridData": { - "h": 6, + "h": 7, "i": "1", - "w": 19, - "x": 10, + "w": 17, + "x": 7, "y": 0 }, "panelIndex": "1", - "panelRefName": "panel_0", + "panelRefName": "panel_1", "title": "Database Connections", - "version": "7.3.0" + "type": "visualization", + "version": "7.11.0" }, { "embeddableConfig": { - "title": "Insert Latency in Milliseconds" + "enhancements": {} }, "gridData": { - "h": 10, - "i": "3", - "w": 24, + "h": 8, + "i": "5", + "w": 17, + "x": 7, + "y": 7 + }, + "panelIndex": "5", + "panelRefName": "panel_5", + "title": "Transaction Blocked", + "type": "visualization", + "version": "7.11.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 15, + "i": "6", + "w": 7, "x": 0, - "y": 6 + "y": 0 }, - "panelIndex": "3", - "panelRefName": "panel_1", - "title": "Insert Latency in Milliseconds", - "version": "7.3.0" + "panelIndex": "6", + "panelRefName": "panel_6", + "title": "AWS Region Filter", + "type": "visualization", + "version": "7.11.0" }, { "embeddableConfig": { - "title": "Select Latency in Milliseconds" + "enhancements": {}, + "hidePanelTitles": false }, "gridData": { - "h": 10, - "i": "4", + "h": 15, + "i": "4d89e43f-299c-4f43-bde2-0ada0983ff23", "w": 24, "x": 24, - "y": 6 + "y": 0 }, - "panelIndex": "4", - "panelRefName": "panel_2", - "title": "Select Latency in Milliseconds", - "version": "7.3.0" + "panelIndex": "4d89e43f-299c-4f43-bde2-0ada0983ff23", + "panelRefName": "panel_4d89e43f-299c-4f43-bde2-0ada0983ff23", + "title": "Free Storage", + "type": "lens", + "version": "7.11.0" }, { "embeddableConfig": { - "title": "Transaction Blocked" + "enhancements": {}, + "hidePanelTitles": false }, "gridData": { - "h": 6, - "i": "5", - "w": 19, - "x": 29, - "y": 0 + "h": 15, + "i": "d409ab5d-84b5-4ecc-86ae-1f79a882b626", + "w": 24, + "x": 0, + "y": 15 }, - "panelIndex": "5", - "panelRefName": "panel_3", - "title": "Transaction Blocked", - "version": "7.3.0" + "panelIndex": "d409ab5d-84b5-4ecc-86ae-1f79a882b626", + "panelRefName": "panel_d409ab5d-84b5-4ecc-86ae-1f79a882b626", + "title": "Read Latency", + "type": "lens", + "version": "7.11.0" }, { - "embeddableConfig": {}, + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "metricbeat-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "metricbeat-*", + "name": "indexpattern-datasource-layer-75b24975-5ca3-4da5-bc1a-92013a901a21", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "75b24975-5ca3-4da5-bc1a-92013a901a21": { + "columnOrder": [ + "bfa06179-6b6f-43e5-a446-f856ff3e51bf", + "af96ca6c-7ab9-47a3-ad8b-29e1578c0076", + "6a87f496-b929-4d24-aede-325d54fedfa1", + "6a87f496-b929-4d24-aede-325d54fedfa1X0" + ], + "columns": { + "6a87f496-b929-4d24-aede-325d54fedfa1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Write Latency (Milliseconds)", + "operationType": "formula", + "params": { + "formula": "average(aws.rds.latency.write)", + "isFormulaBroken": false + }, + "references": [ + "6a87f496-b929-4d24-aede-325d54fedfa1X0" + ], + "scale": "ratio" + }, + "6a87f496-b929-4d24-aede-325d54fedfa1X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of average(aws.rds.latency.write)", + "operationType": "average", + "scale": "ratio", + "sourceField": "aws.rds.latency.write" + }, + "af96ca6c-7ab9-47a3-ad8b-29e1578c0076": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "bfa06179-6b6f-43e5-a446-f856ff3e51bf": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of aws.rds.db_instance.identifier", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "fallback": true, + "type": "alphabetical" + }, + "orderDirection": "asc", + "otherBucket": true, + "size": 10 + }, + "scale": "ordinal", + "sourceField": "aws.rds.db_instance.identifier" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "layers": [ + { + "accessors": [ + "6a87f496-b929-4d24-aede-325d54fedfa1" + ], + "layerId": "75b24975-5ca3-4da5-bc1a-92013a901a21", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "splitAccessor": "bfa06179-6b6f-43e5-a446-f856ff3e51bf", + "xAccessor": "af96ca6c-7ab9-47a3-ad8b-29e1578c0076" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide", + "yLeftExtent": { + "mode": "full" + }, + "yRightExtent": { + "mode": "full" + } + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false, + "type": "lens" + }, "gridData": { - "h": 6, - "i": "6", - "w": 10, - "x": 0, - "y": 0 + "h": 15, + "i": "1abf12dc-d009-4a02-acd4-463383d32a63", + "w": 24, + "x": 24, + "y": 15 }, - "panelIndex": "6", - "panelRefName": "panel_4", - "version": "7.3.0" + "panelIndex": "1abf12dc-d009-4a02-acd4-463383d32a63", + "panelRefName": "panel_1abf12dc-d009-4a02-acd4-463383d32a63", + "title": "Write Latency", + "type": "lens", + "version": "7.14.0" }, { "embeddableConfig": { - "title": "Insert Throughput in Count/Second" + "enhancements": {}, + "hidePanelTitles": false }, "gridData": { - "h": 11, - "i": "7", + "h": 15, + "i": "c5476b0e-6a44-43e5-8bb4-0795c4d097c1", "w": 24, "x": 0, - "y": 16 + "y": 30 }, - "panelIndex": "7", - "panelRefName": "panel_5", - "title": "Insert Throughput in Count/Second", - "version": "7.3.0" + "panelIndex": "c5476b0e-6a44-43e5-8bb4-0795c4d097c1", + "panelRefName": "panel_c5476b0e-6a44-43e5-8bb4-0795c4d097c1", + "title": "Insert Throughput", + "type": "lens", + "version": "7.11.0" }, { "embeddableConfig": { - "title": "Select Throughput in Count/Second" + "enhancements": {}, + "hidePanelTitles": false }, "gridData": { - "h": 11, - "i": "8", + "h": 15, + "i": "bf74bb77-3503-4682-9f0e-6df0994dce5d", "w": 24, "x": 24, - "y": 16 + "y": 30 }, - "panelIndex": "8", - "panelRefName": "panel_6", - "title": "Select Throughput in Count/Second", - "version": "7.3.0" + "panelIndex": "bf74bb77-3503-4682-9f0e-6df0994dce5d", + "panelRefName": "panel_bf74bb77-3503-4682-9f0e-6df0994dce5d", + "title": "Select Throughput", + "type": "lens", + "version": "7.11.0" }, { "embeddableConfig": { - "title": "Disk Queue Depth" + "attributes": { + "references": [ + { + "id": "metricbeat-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "metricbeat-*", + "name": "indexpattern-datasource-layer-dd0a4706-5286-4976-9bc4-f5e7a4964bf6", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "dd0a4706-5286-4976-9bc4-f5e7a4964bf6": { + "columnOrder": [ + "a2bb9c7a-0ddc-4bf7-ae24-98a535a916cc", + "103900c0-dcfa-416f-a272-6efa09c84fce", + "18e6079e-e955-41d0-8196-d2b932cf1fa6", + "18e6079e-e955-41d0-8196-d2b932cf1fa6X0" + ], + "columns": { + "103900c0-dcfa-416f-a272-6efa09c84fce": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "18e6079e-e955-41d0-8196-d2b932cf1fa6": { + "dataType": "number", + "isBucketed": false, + "label": "average(aws.rds.cpu.total.pct)", + "operationType": "formula", + "params": { + "formula": "average(aws.rds.cpu.total.pct)", + "isFormulaBroken": false + }, + "references": [ + "18e6079e-e955-41d0-8196-d2b932cf1fa6X0" + ], + "scale": "ratio" + }, + "18e6079e-e955-41d0-8196-d2b932cf1fa6X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of average(aws.rds.cpu.total.pct)", + "operationType": "average", + "scale": "ratio", + "sourceField": "aws.rds.cpu.total.pct" + }, + "a2bb9c7a-0ddc-4bf7-ae24-98a535a916cc": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of aws.rds.db_instance.identifier", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "fallback": true, + "type": "alphabetical" + }, + "orderDirection": "asc", + "otherBucket": true, + "size": 10 + }, + "scale": "ordinal", + "sourceField": "aws.rds.db_instance.identifier" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "layers": [ + { + "accessors": [ + "18e6079e-e955-41d0-8196-d2b932cf1fa6" + ], + "layerId": "dd0a4706-5286-4976-9bc4-f5e7a4964bf6", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "splitAccessor": "a2bb9c7a-0ddc-4bf7-ae24-98a535a916cc", + "xAccessor": "103900c0-dcfa-416f-a272-6efa09c84fce" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide", + "yLeftExtent": { + "mode": "full" + }, + "yRightExtent": { + "mode": "full" + } + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false, + "type": "lens" }, "gridData": { - "h": 12, - "i": "132653bc-2669-4e8c-b536-06c680e9acf0", - "w": 48, + "h": 15, + "i": "249ff0a6-3fd3-4935-85c3-0c3222d3c498", + "w": 24, "x": 0, - "y": 27 + "y": 45 }, - "panelIndex": "132653bc-2669-4e8c-b536-06c680e9acf0", - "panelRefName": "panel_7", + "panelIndex": "249ff0a6-3fd3-4935-85c3-0c3222d3c498", + "panelRefName": "panel_249ff0a6-3fd3-4935-85c3-0c3222d3c498", + "title": "CPU Total Pct", + "type": "lens", + "version": "7.14.0" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "metricbeat-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "metricbeat-*", + "name": "indexpattern-datasource-layer-14d4ba6b-f4e1-4d40-818a-6aa829d90422", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "14d4ba6b-f4e1-4d40-818a-6aa829d90422": { + "columnOrder": [ + "8954842c-4056-46ef-adfc-29dfc3b0cbd3", + "40493df1-c805-49eb-8dfa-9ff81f7acd4b", + "c7c3ebb2-d611-40a5-aab3-491fa36fe729", + "c7c3ebb2-d611-40a5-aab3-491fa36fe729X0" + ], + "columns": { + "40493df1-c805-49eb-8dfa-9ff81f7acd4b": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "8954842c-4056-46ef-adfc-29dfc3b0cbd3": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of aws.rds.db_instance.identifier", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "fallback": true, + "type": "alphabetical" + }, + "orderDirection": "asc", + "otherBucket": true, + "size": 10 + }, + "scale": "ordinal", + "sourceField": "aws.rds.db_instance.identifier" + }, + "c7c3ebb2-d611-40a5-aab3-491fa36fe729": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Queue Depth (Count)", + "operationType": "formula", + "params": { + "formula": "average(aws.rds.disk_queue_depth)", + "isFormulaBroken": false + }, + "references": [ + "c7c3ebb2-d611-40a5-aab3-491fa36fe729X0" + ], + "scale": "ratio" + }, + "c7c3ebb2-d611-40a5-aab3-491fa36fe729X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Queue Depth (Count)", + "operationType": "average", + "scale": "ratio", + "sourceField": "aws.rds.disk_queue_depth" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "layers": [ + { + "accessors": [ + "c7c3ebb2-d611-40a5-aab3-491fa36fe729" + ], + "layerId": "14d4ba6b-f4e1-4d40-818a-6aa829d90422", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "splitAccessor": "8954842c-4056-46ef-adfc-29dfc3b0cbd3", + "xAccessor": "40493df1-c805-49eb-8dfa-9ff81f7acd4b" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide", + "yLeftExtent": { + "mode": "full" + }, + "yRightExtent": { + "mode": "full" + } + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false, + "type": "lens" + }, + "gridData": { + "h": 15, + "i": "c28488ce-a20e-447f-9a68-ba49b542ab0a", + "w": 24, + "x": 24, + "y": 45 + }, + "panelIndex": "c28488ce-a20e-447f-9a68-ba49b542ab0a", + "panelRefName": "panel_c28488ce-a20e-447f-9a68-ba49b542ab0a", "title": "Disk Queue Depth", - "version": "7.3.0" + "type": "lens", + "version": "7.14.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 15, + "i": "29549114-6ebf-4047-aa56-bc035f66d3b4", + "w": 24, + "x": 0, + "y": 60 + }, + "panelIndex": "29549114-6ebf-4047-aa56-bc035f66d3b4", + "panelRefName": "panel_29549114-6ebf-4047-aa56-bc035f66d3b4", + "title": "Write IOPS", + "type": "lens", + "version": "7.11.0" + }, + { + "embeddableConfig": { + "attributes": { + "references": [ + { + "id": "metricbeat-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "metricbeat-*", + "name": "indexpattern-datasource-layer-e2611df6-ca73-4d53-b0b5-afd8b718c369", + "type": "index-pattern" + } + ], + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "e2611df6-ca73-4d53-b0b5-afd8b718c369": { + "columnOrder": [ + "53a07fa4-b348-44c7-b644-83f3617e5b5c", + "b9e82720-e098-4dd7-ac5b-f3becccd344a", + "85528f23-48f2-462f-8075-eaddd94b21f2" + ], + "columns": { + "53a07fa4-b348-44c7-b644-83f3617e5b5c": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Read IOPS (Count/Second)", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "columnId": "85528f23-48f2-462f-8075-eaddd94b21f2", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "size": 10 + }, + "scale": "ordinal", + "sourceField": "aws.rds.db_instance.identifier" + }, + "85528f23-48f2-462f-8075-eaddd94b21f2": { + "dataType": "number", + "isBucketed": false, + "label": "Average of aws.rds.read_io.ops_per_sec", + "operationType": "average", + "scale": "ratio", + "sourceField": "aws.rds.read_io.ops_per_sec" + }, + "b9e82720-e098-4dd7-ac5b-f3becccd344a": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "layers": [ + { + "accessors": [ + "85528f23-48f2-462f-8075-eaddd94b21f2" + ], + "layerId": "e2611df6-ca73-4d53-b0b5-afd8b718c369", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "splitAccessor": "53a07fa4-b348-44c7-b644-83f3617e5b5c", + "xAccessor": "b9e82720-e098-4dd7-ac5b-f3becccd344a" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide", + "yLeftExtent": { + "mode": "full" + }, + "yRightExtent": { + "mode": "full" + } + } + }, + "title": "", + "type": "lens", + "visualizationType": "lnsXY" + }, + "enhancements": {}, + "hidePanelTitles": false, + "type": "lens" + }, + "gridData": { + "h": 15, + "i": "addd441f-fa2b-4725-8015-619ee176ed0a", + "w": 24, + "x": 24, + "y": 60 + }, + "panelIndex": "addd441f-fa2b-4725-8015-619ee176ed0a", + "panelRefName": "panel_addd441f-fa2b-4725-8015-619ee176ed0a", + "title": "Read IOPS", + "type": "lens", + "version": "7.14.0" } ], "timeRestore": false, "title": "[Metrics AWS] RDS Overview", "version": 1 }, + "coreMigrationVersion": "7.15.0", "id": "aws-3367c170-921f-11e9-aa19-159bf182e06f", + "migrationVersion": { + "dashboard": "7.14.0" + }, "references": [ { "id": "aws-17fcda50-921b-11e9-aa19-159bf182e06f", - "name": "panel_0", + "name": "1:panel_1", "type": "visualization" }, { - "id": "aws-8b8a7f80-921c-11e9-aa19-159bf182e06f", - "name": "panel_1", + "id": "aws-00b29040-921d-11e9-aa19-159bf182e06f", + "name": "5:panel_5", "type": "visualization" }, { - "id": "aws-cc3a1950-921c-11e9-aa19-159bf182e06f", - "name": "panel_2", + "id": "aws-b5308940-7347-11e9-816b-07687310a99a", + "name": "6:panel_6", "type": "visualization" }, { - "id": "aws-00b29040-921d-11e9-aa19-159bf182e06f", - "name": "panel_3", - "type": "visualization" + "id": "aws-68f36fb0-f0a0-11eb-b61f-a53cb4913361", + "name": "4d89e43f-299c-4f43-bde2-0ada0983ff23:panel_4d89e43f-299c-4f43-bde2-0ada0983ff23", + "type": "lens" }, { - "id": "aws-b5308940-7347-11e9-816b-07687310a99a", - "name": "panel_4", - "type": "visualization" + "id": "aws-f9ff3450-f094-11eb-b61f-a53cb4913361", + "name": "d409ab5d-84b5-4ecc-86ae-1f79a882b626:panel_d409ab5d-84b5-4ecc-86ae-1f79a882b626", + "type": "lens" }, { - "id": "aws-c1afd130-921e-11e9-aa19-159bf182e06f", - "name": "panel_5", - "type": "visualization" + "id": "aws-8560b400-f096-11eb-b61f-a53cb4913361", + "name": "1abf12dc-d009-4a02-acd4-463383d32a63:panel_1abf12dc-d009-4a02-acd4-463383d32a63", + "type": "lens" }, { - "id": "aws-e06e4cf0-921e-11e9-aa19-159bf182e06f", - "name": "panel_6", - "type": "visualization" + "id": "metricbeat-*", + "name": "1abf12dc-d009-4a02-acd4-463383d32a63:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" }, { - "id": "aws-966ae990-d979-11e9-9458-bbef63ad717b", - "name": "panel_7", - "type": "visualization" + "id": "metricbeat-*", + "name": "1abf12dc-d009-4a02-acd4-463383d32a63:indexpattern-datasource-layer-75b24975-5ca3-4da5-bc1a-92013a901a21", + "type": "index-pattern" + }, + { + "id": "aws-b0077d10-f09b-11eb-b61f-a53cb4913361", + "name": "c5476b0e-6a44-43e5-8bb4-0795c4d097c1:panel_c5476b0e-6a44-43e5-8bb4-0795c4d097c1", + "type": "lens" + }, + { + "id": "aws-41e37710-f09d-11eb-b61f-a53cb4913361", + "name": "bf74bb77-3503-4682-9f0e-6df0994dce5d:panel_bf74bb77-3503-4682-9f0e-6df0994dce5d", + "type": "lens" + }, + { + "id": "aws-c4419a90-f091-11eb-b61f-a53cb4913361", + "name": "249ff0a6-3fd3-4935-85c3-0c3222d3c498:panel_249ff0a6-3fd3-4935-85c3-0c3222d3c498", + "type": "lens" + }, + { + "id": "metricbeat-*", + "name": "249ff0a6-3fd3-4935-85c3-0c3222d3c498:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "metricbeat-*", + "name": "249ff0a6-3fd3-4935-85c3-0c3222d3c498:indexpattern-datasource-layer-dd0a4706-5286-4976-9bc4-f5e7a4964bf6", + "type": "index-pattern" + }, + { + "id": "aws-bbc80900-f09e-11eb-b61f-a53cb4913361", + "name": "c28488ce-a20e-447f-9a68-ba49b542ab0a:panel_c28488ce-a20e-447f-9a68-ba49b542ab0a", + "type": "lens" + }, + { + "id": "metricbeat-*", + "name": "c28488ce-a20e-447f-9a68-ba49b542ab0a:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "metricbeat-*", + "name": "c28488ce-a20e-447f-9a68-ba49b542ab0a:indexpattern-datasource-layer-14d4ba6b-f4e1-4d40-818a-6aa829d90422", + "type": "index-pattern" + }, + { + "id": "aws-30fd4e40-f0a1-11eb-b61f-a53cb4913361", + "name": "29549114-6ebf-4047-aa56-bc035f66d3b4:panel_29549114-6ebf-4047-aa56-bc035f66d3b4", + "type": "lens" + }, + { + "id": "aws-f757eba0-f0a0-11eb-b61f-a53cb4913361", + "name": "addd441f-fa2b-4725-8015-619ee176ed0a:panel_addd441f-fa2b-4725-8015-619ee176ed0a", + "type": "lens" + }, + { + "id": "metricbeat-*", + "name": "addd441f-fa2b-4725-8015-619ee176ed0a:indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "metricbeat-*", + "name": "addd441f-fa2b-4725-8015-619ee176ed0a:indexpattern-datasource-layer-e2611df6-ca73-4d53-b0b5-afd8b718c369", + "type": "index-pattern" } ], "type": "dashboard" diff --git a/test/packages/aws/kibana/dashboard/aws-80ed1380-41a6-11ec-a605-bff67d9b7872.json b/test/packages/aws/kibana/dashboard/aws-80ed1380-41a6-11ec-a605-bff67d9b7872.json new file mode 100644 index 0000000000..af4d6eb646 --- /dev/null +++ b/test/packages/aws/kibana/dashboard/aws-80ed1380-41a6-11ec-a605-bff67d9b7872.json @@ -0,0 +1,747 @@ +{ + "attributes": { + "description": "Overview of AWS S3 Storage Lens Metrics", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "syncColors": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "enhancements": {}, + "hidePanelTitles": true, + "savedVis": { + "data": { + "aggs": [], + "searchSource": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "description": "", + "params": { + "fontSize": 16, + "markdown": "Daily Overview", + "openLinksInNewTab": false + }, + "title": "", + "type": "markdown", + "uiState": {} + } + }, + "gridData": { + "h": 4, + "i": "0b61b236-11aa-4040-abf9-7b8eb4db1e31", + "w": 48, + "x": 0, + "y": 0 + }, + "panelIndex": "0b61b236-11aa-4040-abf9-7b8eb4db1e31", + "panelRefName": "panel_0b61b236-11aa-4040-abf9-7b8eb4db1e31", + "type": "visualization", + "version": "7.15.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 11, + "i": "9b8691d3-9f30-4776-bf5c-342900818a80", + "w": 8, + "x": 0, + "y": 4 + }, + "panelIndex": "9b8691d3-9f30-4776-bf5c-342900818a80", + "panelRefName": "panel_9b8691d3-9f30-4776-bf5c-342900818a80", + "title": "Filters", + "type": "visualization", + "version": "7.15.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "enhancements": {}, + "hidePanelTitles": true, + "savedVis": { + "data": { + "aggs": [], + "searchSource": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "description": "", + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "background_color_rules": [ + { + "id": "65c0c4b0-42a2-11ec-afe1-bb42ec488e9b" + } + ], + "drop_last_bucket": 0, + "id": "142ea8b7-6859-4f47-a442-71e4c5995e8d", + "index_pattern_ref_name": "metrics_3fc410ab-c8ca-4a7d-9566-c9921f7f3323_0_index_pattern", + "interval": "", + "isModelInvalid": false, + "max_lines_legend": 1, + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "filter": { + "language": "kuery", + "query": "aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" " + }, + "formatter": "bytes", + "id": "de4a05bf-2a2d-422d-98ba-49350f7442d9", + "label": "Total Storage", + "line_width": 1, + "metrics": [ + { + "field": "aws.s3_storage_lens.metrics.StorageBytes.avg", + "id": "cf9f8d20-f581-4036-8f5f-07748b6f9533", + "type": "sum" + } + ], + "override_index_pattern": 1, + "palette": { + "name": "default", + "type": "palette" + }, + "point_size": 1, + "separate_axis": 0, + "series_index_pattern_ref_name": "metrics_3fc410ab-c8ca-4a7d-9566-c9921f7f3323_1_index_pattern", + "series_interval": "3d", + "split_mode": "everything", + "stacked": "none", + "time_range_mode": "last_value" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "", + "time_range_mode": "entire_time_range", + "tooltip_mode": "show_all", + "truncate_legend": 1, + "type": "metric", + "use_kibana_indexes": true + }, + "title": "", + "type": "metrics", + "uiState": {} + } + }, + "gridData": { + "h": 11, + "i": "3fc410ab-c8ca-4a7d-9566-c9921f7f3323", + "w": 8, + "x": 8, + "y": 4 + }, + "panelIndex": "3fc410ab-c8ca-4a7d-9566-c9921f7f3323", + "panelRefName": "panel_3fc410ab-c8ca-4a7d-9566-c9921f7f3323", + "type": "visualization", + "version": "7.15.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "enhancements": {}, + "hidePanelTitles": true + }, + "gridData": { + "h": 11, + "i": "6d7c0316-d2fa-4e3c-9028-fc7f109a7337", + "w": 8, + "x": 16, + "y": 4 + }, + "panelIndex": "6d7c0316-d2fa-4e3c-9028-fc7f109a7337", + "panelRefName": "panel_6d7c0316-d2fa-4e3c-9028-fc7f109a7337", + "type": "visualization", + "version": "7.15.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "enhancements": {}, + "hidePanelTitles": true + }, + "gridData": { + "h": 11, + "i": "90e351eb-70b7-4a7b-b113-b399adf6ff28", + "w": 8, + "x": 24, + "y": 4 + }, + "panelIndex": "90e351eb-70b7-4a7b-b113-b399adf6ff28", + "panelRefName": "panel_90e351eb-70b7-4a7b-b113-b399adf6ff28", + "type": "visualization", + "version": "7.15.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "enhancements": {}, + "hidePanelTitles": true + }, + "gridData": { + "h": 11, + "i": "f5ccf3a7-2922-43ff-bc79-1cd7d56e89a0", + "w": 8, + "x": 32, + "y": 4 + }, + "panelIndex": "f5ccf3a7-2922-43ff-bc79-1cd7d56e89a0", + "panelRefName": "panel_f5ccf3a7-2922-43ff-bc79-1cd7d56e89a0", + "type": "visualization", + "version": "7.15.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "enhancements": {}, + "hidePanelTitles": true + }, + "gridData": { + "h": 11, + "i": "03a07a48-e7e7-4aad-9b3f-74617467c739", + "w": 8, + "x": 40, + "y": 4 + }, + "panelIndex": "03a07a48-e7e7-4aad-9b3f-74617467c739", + "panelRefName": "panel_03a07a48-e7e7-4aad-9b3f-74617467c739", + "type": "visualization", + "version": "7.15.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "enhancements": {}, + "hidePanelTitles": false, + "savedVis": { + "data": { + "aggs": [], + "searchSource": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "description": "", + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "background_color_rules": [ + { + "id": "88f52970-42a8-11ec-afe1-bb42ec488e9b" + } + ], + "bar_color_rules": [ + { + "id": "5fac2960-42a8-11ec-afe1-bb42ec488e9b" + } + ], + "drop_last_bucket": 0, + "filter": { + "language": "kuery", + "query": "aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" " + }, + "gauge_color_rules": [ + { + "id": "864db020-42a8-11ec-afe1-bb42ec488e9b" + } + ], + "gauge_inner_width": 10, + "gauge_style": "half", + "gauge_width": 10, + "id": "c08f117f-f0e8-4d10-ace7-5d2a5aab35c0", + "index_pattern_ref_name": "metrics_82b50202-e42d-49fb-9db7-2213216350f9_0_index_pattern", + "interval": "3d", + "isModelInvalid": false, + "max_lines_legend": 1, + "series": [ + { + "axis_position": "left", + "chart_type": "bar", + "color": "rgba(96,146,192,1)", + "fill": "1", + "formatter": "bytes", + "id": "de4a05bf-2a2d-422d-98ba-49350f7442d9", + "label": "Total Storage", + "line_width": 1, + "metrics": [ + { + "field": "aws.s3_storage_lens.metrics.StorageBytes.avg", + "id": "cf9f8d20-f581-4036-8f5f-07748b6f9533", + "type": "sum" + } + ], + "palette": { + "name": "default", + "type": "palette" + }, + "point_size": 1, + "separate_axis": 1, + "split_mode": "terms", + "stacked": "none", + "terms_field": "aws.dimensions.aws_region", + "terms_order_by": "cf9f8d20-f581-4036-8f5f-07748b6f9533", + "terms_size": "10", + "type": "timeseries" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "", + "time_range_mode": "last_value", + "tooltip_mode": "show_all", + "truncate_legend": 1, + "type": "top_n", + "use_kibana_indexes": true + }, + "title": "", + "type": "metrics", + "uiState": {} + } + }, + "gridData": { + "h": 7, + "i": "82b50202-e42d-49fb-9db7-2213216350f9", + "w": 24, + "x": 0, + "y": 15 + }, + "panelIndex": "82b50202-e42d-49fb-9db7-2213216350f9", + "panelRefName": "panel_82b50202-e42d-49fb-9db7-2213216350f9", + "title": "Total Storage Region Distribution", + "type": "visualization", + "version": "7.15.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 7, + "i": "048f0020-8699-459c-bbb3-33a5597798f9", + "w": 24, + "x": 24, + "y": 15 + }, + "panelIndex": "048f0020-8699-459c-bbb3-33a5597798f9", + "panelRefName": "panel_048f0020-8699-459c-bbb3-33a5597798f9", + "title": "Total Storage Class Distribution", + "type": "visualization", + "version": "7.15.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 8, + "i": "939563b6-0601-45ef-86fc-bc18fb7fa474", + "w": 24, + "x": 0, + "y": 22 + }, + "panelIndex": "939563b6-0601-45ef-86fc-bc18fb7fa474", + "panelRefName": "panel_939563b6-0601-45ef-86fc-bc18fb7fa474", + "title": "Object Count Region Distribution", + "type": "visualization", + "version": "7.15.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 8, + "i": "283f4fe8-710f-4f21-b024-ecb77d1933ab", + "w": 24, + "x": 24, + "y": 22 + }, + "panelIndex": "283f4fe8-710f-4f21-b024-ecb77d1933ab", + "panelRefName": "panel_283f4fe8-710f-4f21-b024-ecb77d1933ab", + "title": "Object Count Class Distribution", + "type": "visualization", + "version": "7.15.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "enhancements": {}, + "hidePanelTitles": true, + "savedVis": { + "data": { + "aggs": [], + "searchSource": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "description": "", + "params": { + "fontSize": 16, + "markdown": "Trends", + "openLinksInNewTab": false + }, + "title": "", + "type": "markdown", + "uiState": {} + } + }, + "gridData": { + "h": 4, + "i": "826b9a1d-3ff4-4792-a833-e274f1a39c46", + "w": 48, + "x": 0, + "y": 30 + }, + "panelIndex": "826b9a1d-3ff4-4792-a833-e274f1a39c46", + "panelRefName": "panel_826b9a1d-3ff4-4792-a833-e274f1a39c46", + "type": "visualization", + "version": "7.15.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 14, + "i": "02d4d942-8c9a-4cb4-b3a3-18aacc0b2493", + "w": 48, + "x": 0, + "y": 34 + }, + "panelIndex": "02d4d942-8c9a-4cb4-b3a3-18aacc0b2493", + "panelRefName": "panel_02d4d942-8c9a-4cb4-b3a3-18aacc0b2493", + "title": "Total Storage and Object Count", + "type": "lens", + "version": "7.15.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "enhancements": {}, + "hidePanelTitles": true, + "savedVis": { + "data": { + "aggs": [], + "searchSource": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "description": "", + "params": { + "fontSize": 14, + "markdown": "Cost Efficiency", + "openLinksInNewTab": false + }, + "title": "", + "type": "markdown", + "uiState": {} + } + }, + "gridData": { + "h": 4, + "i": "45489a96-fc7f-4c8f-b037-2a6a7fa37316", + "w": 48, + "x": 0, + "y": 48 + }, + "panelIndex": "45489a96-fc7f-4c8f-b037-2a6a7fa37316", + "panelRefName": "panel_45489a96-fc7f-4c8f-b037-2a6a7fa37316", + "type": "visualization", + "version": "7.15.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "enhancements": {}, + "hidePanelTitles": true + }, + "gridData": { + "h": 16, + "i": "d77a360f-abbd-41cb-8c72-ac1848168dbc", + "w": 6, + "x": 0, + "y": 52 + }, + "panelIndex": "d77a360f-abbd-41cb-8c72-ac1848168dbc", + "panelRefName": "panel_d77a360f-abbd-41cb-8c72-ac1848168dbc", + "type": "visualization", + "version": "7.15.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 16, + "i": "4a0a8cb0-acce-4b4f-8635-d19b33a5b7c7", + "w": 21, + "x": 6, + "y": 52 + }, + "panelIndex": "4a0a8cb0-acce-4b4f-8635-d19b33a5b7c7", + "panelRefName": "panel_4a0a8cb0-acce-4b4f-8635-d19b33a5b7c7", + "title": "Current Version Bytes Percentage", + "type": "lens", + "version": "7.15.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 16, + "i": "f4087f7c-4714-430d-9fce-9232215efcea", + "w": 21, + "x": 27, + "y": 52 + }, + "panelIndex": "f4087f7c-4714-430d-9fce-9232215efcea", + "panelRefName": "panel_f4087f7c-4714-430d-9fce-9232215efcea", + "title": "Incomplete MPU bytes Percentage", + "type": "lens", + "version": "7.15.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "enhancements": {}, + "hidePanelTitles": true + }, + "gridData": { + "h": 4, + "i": "2dbf68cd-48be-4e1f-a526-a47ec24f8359", + "w": 48, + "x": 0, + "y": 68 + }, + "panelIndex": "2dbf68cd-48be-4e1f-a526-a47ec24f8359", + "panelRefName": "panel_2dbf68cd-48be-4e1f-a526-a47ec24f8359", + "type": "visualization", + "version": "7.15.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 11, + "i": "b21580ab-7ec7-47fe-9449-14b1d617fd63", + "w": 17, + "x": 0, + "y": 72 + }, + "panelIndex": "b21580ab-7ec7-47fe-9449-14b1d617fd63", + "panelRefName": "panel_b21580ab-7ec7-47fe-9449-14b1d617fd63", + "title": "Object Lock Bytes Percentage", + "type": "visualization", + "version": "7.15.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 11, + "i": "ba2bd05a-ad1c-4d35-8396-89febc950636", + "w": 16, + "x": 17, + "y": 72 + }, + "panelIndex": "ba2bd05a-ad1c-4d35-8396-89febc950636", + "panelRefName": "panel_ba2bd05a-ad1c-4d35-8396-89febc950636", + "title": "Replicated Bytes Percentage", + "type": "visualization", + "version": "7.15.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "enhancements": {}, + "hidePanelTitles": false + }, + "gridData": { + "h": 11, + "i": "c8ab3a47-5316-495a-bb9e-a78fb867b059", + "w": 15, + "x": 33, + "y": 72 + }, + "panelIndex": "c8ab3a47-5316-495a-bb9e-a78fb867b059", + "panelRefName": "panel_c8ab3a47-5316-495a-bb9e-a78fb867b059", + "title": "Encrypted Bytes Percentage", + "type": "visualization", + "version": "7.15.0-SNAPSHOT" + } + ], + "timeRestore": false, + "title": "[Metrics AWS] S3 Storage Lens Overview", + "version": 1 + }, + "coreMigrationVersion": "7.15.0", + "id": "aws-80ed1380-41a6-11ec-a605-bff67d9b7872", + "migrationVersion": { + "dashboard": "7.15.0" + }, + "references": [ + { + "id": "aws-82b8f4a0-427c-11ec-9b39-b9df88395505", + "name": "0b61b236-11aa-4040-abf9-7b8eb4db1e31:panel_0b61b236-11aa-4040-abf9-7b8eb4db1e31", + "type": "visualization" + }, + { + "id": "aws-57b610a0-41ad-11ec-a605-bff67d9b7872", + "name": "9b8691d3-9f30-4776-bf5c-342900818a80:panel_9b8691d3-9f30-4776-bf5c-342900818a80", + "type": "visualization" + }, + { + "id": "aws-3c08b9b0-42a3-11ec-a440-675e476c18bb", + "name": "3fc410ab-c8ca-4a7d-9566-c9921f7f3323:panel_3fc410ab-c8ca-4a7d-9566-c9921f7f3323", + "type": "visualization" + }, + { + "id": "metrics-*", + "name": "3fc410ab-c8ca-4a7d-9566-c9921f7f3323:metrics_3fc410ab-c8ca-4a7d-9566-c9921f7f3323_0_index_pattern", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "3fc410ab-c8ca-4a7d-9566-c9921f7f3323:metrics_3fc410ab-c8ca-4a7d-9566-c9921f7f3323_1_index_pattern", + "type": "index-pattern" + }, + { + "id": "aws-a1670af0-42a3-11ec-a440-675e476c18bb", + "name": "6d7c0316-d2fa-4e3c-9028-fc7f109a7337:panel_6d7c0316-d2fa-4e3c-9028-fc7f109a7337", + "type": "visualization" + }, + { + "id": "aws-2e265070-42a6-11ec-a440-675e476c18bb", + "name": "90e351eb-70b7-4a7b-b113-b399adf6ff28:panel_90e351eb-70b7-4a7b-b113-b399adf6ff28", + "type": "visualization" + }, + { + "id": "aws-526a1ba0-42a7-11ec-a440-675e476c18bb", + "name": "f5ccf3a7-2922-43ff-bc79-1cd7d56e89a0:panel_f5ccf3a7-2922-43ff-bc79-1cd7d56e89a0", + "type": "visualization" + }, + { + "id": "aws-96834640-42a7-11ec-a440-675e476c18bb", + "name": "03a07a48-e7e7-4aad-9b3f-74617467c739:panel_03a07a48-e7e7-4aad-9b3f-74617467c739", + "type": "visualization" + }, + { + "id": "aws-11f4e0d0-42a9-11ec-a440-675e476c18bb", + "name": "82b50202-e42d-49fb-9db7-2213216350f9:panel_82b50202-e42d-49fb-9db7-2213216350f9", + "type": "visualization" + }, + { + "id": "metrics-*", + "name": "82b50202-e42d-49fb-9db7-2213216350f9:metrics_82b50202-e42d-49fb-9db7-2213216350f9_0_index_pattern", + "type": "index-pattern" + }, + { + "id": "aws-e1c345e0-42a9-11ec-a440-675e476c18bb", + "name": "048f0020-8699-459c-bbb3-33a5597798f9:panel_048f0020-8699-459c-bbb3-33a5597798f9", + "type": "visualization" + }, + { + "id": "aws-37aeedc0-42a9-11ec-a440-675e476c18bb", + "name": "939563b6-0601-45ef-86fc-bc18fb7fa474:panel_939563b6-0601-45ef-86fc-bc18fb7fa474", + "type": "visualization" + }, + { + "id": "aws-f9b2beb0-42a9-11ec-a440-675e476c18bb", + "name": "283f4fe8-710f-4f21-b024-ecb77d1933ab:panel_283f4fe8-710f-4f21-b024-ecb77d1933ab", + "type": "visualization" + }, + { + "id": "aws-7593a130-427c-11ec-9b39-b9df88395505", + "name": "826b9a1d-3ff4-4792-a833-e274f1a39c46:panel_826b9a1d-3ff4-4792-a833-e274f1a39c46", + "type": "visualization" + }, + { + "id": "aws-1a1508a0-41a8-11ec-a605-bff67d9b7872", + "name": "02d4d942-8c9a-4cb4-b3a3-18aacc0b2493:panel_02d4d942-8c9a-4cb4-b3a3-18aacc0b2493", + "type": "lens" + }, + { + "id": "aws-5c586e80-427c-11ec-9b39-b9df88395505", + "name": "45489a96-fc7f-4c8f-b037-2a6a7fa37316:panel_45489a96-fc7f-4c8f-b037-2a6a7fa37316", + "type": "visualization" + }, + { + "id": "aws-41f5aaf0-42ac-11ec-a440-675e476c18bb", + "name": "d77a360f-abbd-41cb-8c72-ac1848168dbc:panel_d77a360f-abbd-41cb-8c72-ac1848168dbc", + "type": "visualization" + }, + { + "id": "aws-9fb44b60-41b2-11ec-a605-bff67d9b7872", + "name": "4a0a8cb0-acce-4b4f-8635-d19b33a5b7c7:panel_4a0a8cb0-acce-4b4f-8635-d19b33a5b7c7", + "type": "lens" + }, + { + "id": "aws-352dd3f0-41b3-11ec-a605-bff67d9b7872", + "name": "f4087f7c-4714-430d-9fce-9232215efcea:panel_f4087f7c-4714-430d-9fce-9232215efcea", + "type": "lens" + }, + { + "id": "aws-ef9717b0-427b-11ec-9b39-b9df88395505", + "name": "2dbf68cd-48be-4e1f-a526-a47ec24f8359:panel_2dbf68cd-48be-4e1f-a526-a47ec24f8359", + "type": "visualization" + }, + { + "id": "aws-c4a82470-42aa-11ec-a440-675e476c18bb", + "name": "b21580ab-7ec7-47fe-9449-14b1d617fd63:panel_b21580ab-7ec7-47fe-9449-14b1d617fd63", + "type": "visualization" + }, + { + "id": "aws-7dd7fab0-42ab-11ec-a440-675e476c18bb", + "name": "ba2bd05a-ad1c-4d35-8396-89febc950636:panel_ba2bd05a-ad1c-4d35-8396-89febc950636", + "type": "visualization" + }, + { + "id": "aws-c631dc40-42ab-11ec-a440-675e476c18bb", + "name": "c8ab3a47-5316-495a-bb9e-a78fb867b059:panel_c8ab3a47-5316-495a-bb9e-a78fb867b059", + "type": "visualization" + } + ], + "type": "dashboard" +} \ No newline at end of file diff --git a/test/packages/aws/kibana/lens/aws-0d511340-41a1-11ec-a605-bff67d9b7872.json b/test/packages/aws/kibana/lens/aws-0d511340-41a1-11ec-a605-bff67d9b7872.json new file mode 100644 index 0000000000..469baadaf5 --- /dev/null +++ b/test/packages/aws/kibana/lens/aws-0d511340-41a1-11ec-a605-bff67d9b7872.json @@ -0,0 +1,107 @@ +{ + "attributes": { + "description": "", + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "eab1960c-2b9a-4e4a-9380-c29e91cbb47f": { + "columnOrder": [ + "b7788a69-bb42-4dc4-b7a0-dfc01d99236f", + "b7788a69-bb42-4dc4-b7a0-dfc01d99236fX0" + ], + "columns": { + "b7788a69-bb42-4dc4-b7a0-dfc01d99236f": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Buckets", + "operationType": "formula", + "params": { + "formula": "unique_count(aws.dimensions.bucket_name, kql='not aws.dimensions.storage_class : \"-\" ')", + "isFormulaBroken": false + }, + "references": [ + "b7788a69-bb42-4dc4-b7a0-dfc01d99236fX0" + ], + "scale": "ratio" + }, + "b7788a69-bb42-4dc4-b7a0-dfc01d99236fX0": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "not aws.dimensions.storage_class : \"-\" " + }, + "isBucketed": false, + "label": "Part of unique_count(aws.dimensions.bucket_name, kql='not aws.dimensions.storage_class : \"-\" ')", + "operationType": "unique_count", + "scale": "ratio", + "sourceField": "aws.dimensions.bucket_name" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "indexRefName": "filter-index-pattern-0", + "key": "data_stream.dataset", + "negate": false, + "params": { + "query": "aws.s3_storage_lens" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "data_stream.dataset": "aws.s3_storage_lens" + } + } + } + ], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "accessor": "b7788a69-bb42-4dc4-b7a0-dfc01d99236f", + "layerId": "eab1960c-2b9a-4e4a-9380-c29e91cbb47f", + "layerType": "data" + } + }, + "title": "S3 Storage Lens Buckets [Metrics AWS]", + "visualizationType": "lnsMetric" + }, + "coreMigrationVersion": "7.15.0", + "id": "aws-0d511340-41a1-11ec-a605-bff67d9b7872", + "migrationVersion": { + "lens": "7.15.0" + }, + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-eab1960c-2b9a-4e4a-9380-c29e91cbb47f", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "filter-index-pattern-0", + "type": "index-pattern" + } + ], + "type": "lens" +} \ No newline at end of file diff --git a/test/packages/aws/kibana/lens/aws-1a1508a0-41a8-11ec-a605-bff67d9b7872.json b/test/packages/aws/kibana/lens/aws-1a1508a0-41a8-11ec-a605-bff67d9b7872.json new file mode 100644 index 0000000000..20509beae9 --- /dev/null +++ b/test/packages/aws/kibana/lens/aws-1a1508a0-41a8-11ec-a605-bff67d9b7872.json @@ -0,0 +1,209 @@ +{ + "attributes": { + "description": "", + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "eab1960c-2b9a-4e4a-9380-c29e91cbb47f": { + "columnOrder": [ + "e9704a86-fb68-4316-b885-42328390c6c0", + "a80c40e7-af1b-4ac9-ba2e-baa7c13a729a", + "d3578c4c-8e60-4bb8-9295-72b90c88d168", + "a80c40e7-af1b-4ac9-ba2e-baa7c13a729aX0", + "d3578c4c-8e60-4bb8-9295-72b90c88d168X0" + ], + "columns": { + "a80c40e7-af1b-4ac9-ba2e-baa7c13a729a": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Total Storage", + "operationType": "formula", + "params": { + "format": { + "id": "bytes", + "params": { + "decimals": 2 + } + }, + "formula": "sum(aws.s3_storage_lens.metrics.StorageBytes.avg, kql='aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" ')", + "isFormulaBroken": false + }, + "references": [ + "a80c40e7-af1b-4ac9-ba2e-baa7c13a729aX0" + ], + "scale": "ratio" + }, + "a80c40e7-af1b-4ac9-ba2e-baa7c13a729aX0": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" " + }, + "isBucketed": false, + "label": "Part of Total Storage", + "operationType": "sum", + "scale": "ratio", + "sourceField": "aws.s3_storage_lens.metrics.StorageBytes.avg" + }, + "d3578c4c-8e60-4bb8-9295-72b90c88d168": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Storage Count", + "operationType": "formula", + "params": { + "formula": "sum(aws.s3_storage_lens.metrics.ObjectCount.avg, kql='aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" ')", + "isFormulaBroken": false + }, + "references": [ + "d3578c4c-8e60-4bb8-9295-72b90c88d168X0" + ], + "scale": "ratio" + }, + "d3578c4c-8e60-4bb8-9295-72b90c88d168X0": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" " + }, + "isBucketed": false, + "label": "Part of Storage Count", + "operationType": "sum", + "scale": "ratio", + "sourceField": "aws.s3_storage_lens.metrics.ObjectCount.avg" + }, + "e9704a86-fb68-4316-b885-42328390c6c0": { + "customLabel": false, + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "interval": "1d" + }, + "scale": "interval", + "sourceField": "@timestamp" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "indexRefName": "filter-index-pattern-0", + "key": "data_stream.dataset", + "negate": false, + "params": { + "query": "aws.s3_storage_lens" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "data_stream.dataset": "aws.s3_storage_lens" + } + } + } + ], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "a80c40e7-af1b-4ac9-ba2e-baa7c13a729a", + "d3578c4c-8e60-4bb8-9295-72b90c88d168" + ], + "layerId": "eab1960c-2b9a-4e4a-9380-c29e91cbb47f", + "layerType": "data", + "seriesType": "line", + "xAccessor": "e9704a86-fb68-4316-b885-42328390c6c0", + "yConfig": [ + { + "axisMode": "auto", + "forAccessor": "d3578c4c-8e60-4bb8-9295-72b90c88d168" + }, + { + "axisMode": "auto", + "color": "#e7664c", + "forAccessor": "a80c40e7-af1b-4ac9-ba2e-baa7c13a729a" + } + ] + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide", + "yLeftExtent": { + "mode": "dataBounds" + }, + "yRightExtent": { + "mode": "dataBounds" + } + } + }, + "title": "S3 Storage Lens Total Storage and Object Count [Metrics AWS]", + "visualizationType": "lnsXY" + }, + "coreMigrationVersion": "7.15.0", + "id": "aws-1a1508a0-41a8-11ec-a605-bff67d9b7872", + "migrationVersion": { + "lens": "7.15.0" + }, + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-eab1960c-2b9a-4e4a-9380-c29e91cbb47f", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "filter-index-pattern-0", + "type": "index-pattern" + } + ], + "type": "lens" +} \ No newline at end of file diff --git a/test/packages/aws/kibana/lens/aws-30e707b0-41a1-11ec-a605-bff67d9b7872.json b/test/packages/aws/kibana/lens/aws-30e707b0-41a1-11ec-a605-bff67d9b7872.json new file mode 100644 index 0000000000..13e63818b7 --- /dev/null +++ b/test/packages/aws/kibana/lens/aws-30e707b0-41a1-11ec-a605-bff67d9b7872.json @@ -0,0 +1,87 @@ +{ + "attributes": { + "description": "", + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "eab1960c-2b9a-4e4a-9380-c29e91cbb47f": { + "columnOrder": [ + "a80c40e7-af1b-4ac9-ba2e-baa7c13a729a" + ], + "columns": { + "a80c40e7-af1b-4ac9-ba2e-baa7c13a729a": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Accounts", + "operationType": "unique_count", + "scale": "ratio", + "sourceField": "aws.dimensions.aws_account_number" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "indexRefName": "filter-index-pattern-0", + "key": "data_stream.dataset", + "negate": false, + "params": { + "query": "aws.s3_storage_lens" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "data_stream.dataset": "aws.s3_storage_lens" + } + } + } + ], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "accessor": "a80c40e7-af1b-4ac9-ba2e-baa7c13a729a", + "layerId": "eab1960c-2b9a-4e4a-9380-c29e91cbb47f", + "layerType": "data" + } + }, + "title": "S3 Storage Lens Accounts [Metrics AWS]", + "visualizationType": "lnsMetric" + }, + "coreMigrationVersion": "7.15.0", + "id": "aws-30e707b0-41a1-11ec-a605-bff67d9b7872", + "migrationVersion": { + "lens": "7.15.0" + }, + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-eab1960c-2b9a-4e4a-9380-c29e91cbb47f", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "filter-index-pattern-0", + "type": "index-pattern" + } + ], + "type": "lens" +} \ No newline at end of file diff --git a/test/packages/aws/kibana/lens/aws-30fd4e40-f0a1-11eb-b61f-a53cb4913361.json b/test/packages/aws/kibana/lens/aws-30fd4e40-f0a1-11eb-b61f-a53cb4913361.json new file mode 100644 index 0000000000..a492be7897 --- /dev/null +++ b/test/packages/aws/kibana/lens/aws-30fd4e40-f0a1-11eb-b61f-a53cb4913361.json @@ -0,0 +1,145 @@ +{ + "attributes": { + "description": null, + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "94e6f698-4af3-4acd-a018-867330b4e0de": { + "columnOrder": [ + "ba1bdf55-b2f8-4bb4-b78c-caab170367e0", + "98b1c682-acf5-4331-8129-62177616a221", + "c7534b00-fa2b-4633-84da-83d71de297f8", + "c7534b00-fa2b-4633-84da-83d71de297f8X0" + ], + "columns": { + "98b1c682-acf5-4331-8129-62177616a221": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of aws.rds.db_instance.identifier", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "fallback": true, + "type": "alphabetical" + }, + "orderDirection": "asc", + "otherBucket": true, + "size": 10 + }, + "scale": "ordinal", + "sourceField": "aws.rds.db_instance.identifier" + }, + "ba1bdf55-b2f8-4bb4-b78c-caab170367e0": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "c7534b00-fa2b-4633-84da-83d71de297f8": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Write IOPS (Count/Second)", + "operationType": "formula", + "params": { + "formula": "average(aws.rds.write_io.ops_per_sec)", + "isFormulaBroken": false + }, + "references": [ + "c7534b00-fa2b-4633-84da-83d71de297f8X0" + ], + "scale": "ratio" + }, + "c7534b00-fa2b-4633-84da-83d71de297f8X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Write IOPS (Count/Second)", + "operationType": "average", + "scale": "ratio", + "sourceField": "aws.rds.write_io.ops_per_sec" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "layers": [ + { + "accessors": [ + "c7534b00-fa2b-4633-84da-83d71de297f8" + ], + "layerId": "94e6f698-4af3-4acd-a018-867330b4e0de", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "splitAccessor": "98b1c682-acf5-4331-8129-62177616a221", + "xAccessor": "ba1bdf55-b2f8-4bb4-b78c-caab170367e0" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide", + "yLeftExtent": { + "mode": "full" + }, + "yRightExtent": { + "mode": "full" + } + } + }, + "title": "RDS Write IOPS [Metrics AWS]", + "visualizationType": "lnsXY" + }, + "coreMigrationVersion": "7.15.0", + "id": "aws-30fd4e40-f0a1-11eb-b61f-a53cb4913361", + "migrationVersion": { + "lens": "7.14.0" + }, + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-94e6f698-4af3-4acd-a018-867330b4e0de", + "type": "index-pattern" + } + ], + "type": "lens" +} \ No newline at end of file diff --git a/test/packages/aws/kibana/lens/aws-352dd3f0-41b3-11ec-a605-bff67d9b7872.json b/test/packages/aws/kibana/lens/aws-352dd3f0-41b3-11ec-a605-bff67d9b7872.json new file mode 100644 index 0000000000..a761479abf --- /dev/null +++ b/test/packages/aws/kibana/lens/aws-352dd3f0-41b3-11ec-a605-bff67d9b7872.json @@ -0,0 +1,181 @@ +{ + "attributes": { + "description": "", + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "af6bd20f-099d-4817-a951-99bcba7e1752": { + "columnOrder": [ + "e959c162-4cf1-42f8-bcc7-e08698adc162", + "f13847d9-f404-41a8-8e17-12b47d683bd0", + "f13847d9-f404-41a8-8e17-12b47d683bd0X0", + "f13847d9-f404-41a8-8e17-12b47d683bd0X1", + "f13847d9-f404-41a8-8e17-12b47d683bd0X2" + ], + "columns": { + "e959c162-4cf1-42f8-bcc7-e08698adc162": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "interval": "1d" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "f13847d9-f404-41a8-8e17-12b47d683bd0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "% incomplete MPU bytes", + "operationType": "formula", + "params": { + "format": { + "id": "percent", + "params": { + "decimals": 2 + } + }, + "formula": "divide(sum(aws.s3_storage_lens.metrics.IncompleteMultipartUploadStorageBytes.avg, kql='aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" '), sum(aws.s3_storage_lens.metrics.StorageBytes.avg, kql='aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" '))", + "isFormulaBroken": false + }, + "references": [ + "f13847d9-f404-41a8-8e17-12b47d683bd0X2" + ], + "scale": "ratio" + }, + "f13847d9-f404-41a8-8e17-12b47d683bd0X0": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" " + }, + "isBucketed": false, + "label": "Part of % incomplete MPU bytes", + "operationType": "sum", + "scale": "ratio", + "sourceField": "aws.s3_storage_lens.metrics.IncompleteMultipartUploadStorageBytes.avg" + }, + "f13847d9-f404-41a8-8e17-12b47d683bd0X1": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" " + }, + "isBucketed": false, + "label": "Part of % incomplete MPU bytes", + "operationType": "sum", + "scale": "ratio", + "sourceField": "aws.s3_storage_lens.metrics.StorageBytes.avg" + }, + "f13847d9-f404-41a8-8e17-12b47d683bd0X2": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of % incomplete MPU bytes", + "operationType": "math", + "params": { + "tinymathAst": { + "args": [ + "f13847d9-f404-41a8-8e17-12b47d683bd0X0", + "f13847d9-f404-41a8-8e17-12b47d683bd0X1" + ], + "location": { + "max": 313, + "min": 0 + }, + "name": "divide", + "text": "divide(sum(aws.s3_storage_lens.metrics.IncompleteMultipartUploadStorageBytes.avg, kql='aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" '), sum(aws.s3_storage_lens.metrics.StorageBytes.avg, kql='aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" '))", + "type": "function" + } + }, + "references": [ + "f13847d9-f404-41a8-8e17-12b47d683bd0X0", + "f13847d9-f404-41a8-8e17-12b47d683bd0X1" + ], + "scale": "ratio" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "f13847d9-f404-41a8-8e17-12b47d683bd0" + ], + "layerId": "af6bd20f-099d-4817-a951-99bcba7e1752", + "layerType": "data", + "seriesType": "line", + "xAccessor": "e959c162-4cf1-42f8-bcc7-e08698adc162" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide", + "yLeftExtent": { + "mode": "full" + }, + "yRightExtent": { + "mode": "full" + } + } + }, + "title": "S3 Storage Lens Percentage incomplete MPU bytes [AWS Metrics]", + "visualizationType": "lnsXY" + }, + "coreMigrationVersion": "7.15.0", + "id": "aws-352dd3f0-41b3-11ec-a605-bff67d9b7872", + "migrationVersion": { + "lens": "7.15.0" + }, + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-af6bd20f-099d-4817-a951-99bcba7e1752", + "type": "index-pattern" + } + ], + "type": "lens" +} \ No newline at end of file diff --git a/test/packages/aws/kibana/lens/aws-41e37710-f09d-11eb-b61f-a53cb4913361.json b/test/packages/aws/kibana/lens/aws-41e37710-f09d-11eb-b61f-a53cb4913361.json new file mode 100644 index 0000000000..d353c5ed1f --- /dev/null +++ b/test/packages/aws/kibana/lens/aws-41e37710-f09d-11eb-b61f-a53cb4913361.json @@ -0,0 +1,145 @@ +{ + "attributes": { + "description": "", + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "8682174a-4cff-4d95-b719-1fc306f5b33a": { + "columnOrder": [ + "bc08fa3e-ce15-4acd-a0fd-c5c5c5452441", + "f45a0753-4e23-43c4-80f7-4a9aa9548a6e", + "85980678-0e26-4f77-b735-7ec5ebbc472e", + "85980678-0e26-4f77-b735-7ec5ebbc472eX0" + ], + "columns": { + "85980678-0e26-4f77-b735-7ec5ebbc472e": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Select Throughput Count/Second", + "operationType": "formula", + "params": { + "formula": "average(aws.rds.throughput.select)", + "isFormulaBroken": false + }, + "references": [ + "85980678-0e26-4f77-b735-7ec5ebbc472eX0" + ], + "scale": "ratio" + }, + "85980678-0e26-4f77-b735-7ec5ebbc472eX0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Select Throughput Count/Second", + "operationType": "average", + "scale": "ratio", + "sourceField": "aws.rds.throughput.select" + }, + "bc08fa3e-ce15-4acd-a0fd-c5c5c5452441": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of aws.rds.db_instance.identifier", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "fallback": true, + "type": "alphabetical" + }, + "orderDirection": "asc", + "otherBucket": true, + "size": 10 + }, + "scale": "ordinal", + "sourceField": "aws.rds.db_instance.identifier" + }, + "f45a0753-4e23-43c4-80f7-4a9aa9548a6e": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "layers": [ + { + "accessors": [ + "85980678-0e26-4f77-b735-7ec5ebbc472e" + ], + "layerId": "8682174a-4cff-4d95-b719-1fc306f5b33a", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "splitAccessor": "bc08fa3e-ce15-4acd-a0fd-c5c5c5452441", + "xAccessor": "f45a0753-4e23-43c4-80f7-4a9aa9548a6e" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide", + "yLeftExtent": { + "mode": "full" + }, + "yRightExtent": { + "mode": "full" + } + } + }, + "title": "RDS Select Throughput[Metrics AWS]", + "visualizationType": "lnsXY" + }, + "coreMigrationVersion": "7.15.0", + "id": "aws-41e37710-f09d-11eb-b61f-a53cb4913361", + "migrationVersion": { + "lens": "7.14.0" + }, + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-8682174a-4cff-4d95-b719-1fc306f5b33a", + "type": "index-pattern" + } + ], + "type": "lens" +} \ No newline at end of file diff --git a/test/packages/aws/kibana/lens/aws-6445e130-41b5-11ec-a605-bff67d9b7872.json b/test/packages/aws/kibana/lens/aws-6445e130-41b5-11ec-a605-bff67d9b7872.json new file mode 100644 index 0000000000..5f8dd64ad4 --- /dev/null +++ b/test/packages/aws/kibana/lens/aws-6445e130-41b5-11ec-a605-bff67d9b7872.json @@ -0,0 +1,147 @@ +{ + "attributes": { + "description": "", + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "af6bd20f-099d-4817-a951-99bcba7e1752": { + "columnOrder": [ + "f13847d9-f404-41a8-8e17-12b47d683bd0", + "f13847d9-f404-41a8-8e17-12b47d683bd0X0", + "f13847d9-f404-41a8-8e17-12b47d683bd0X1", + "f13847d9-f404-41a8-8e17-12b47d683bd0X2" + ], + "columns": { + "f13847d9-f404-41a8-8e17-12b47d683bd0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "% Encrypted Bytes", + "operationType": "formula", + "params": { + "format": { + "id": "percent", + "params": { + "decimals": 2 + } + }, + "formula": "divide(sum(aws.s3_storage_lens.metrics.EncryptedStorageBytes.avg), sum(aws.s3_storage_lens.metrics.StorageBytes.avg))", + "isFormulaBroken": false + }, + "references": [ + "f13847d9-f404-41a8-8e17-12b47d683bd0X2" + ], + "scale": "ratio" + }, + "f13847d9-f404-41a8-8e17-12b47d683bd0X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of % Current Version Bytes", + "operationType": "sum", + "scale": "ratio", + "sourceField": "aws.s3_storage_lens.metrics.EncryptedStorageBytes.avg" + }, + "f13847d9-f404-41a8-8e17-12b47d683bd0X1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of % Current Version Bytes", + "operationType": "sum", + "scale": "ratio", + "sourceField": "aws.s3_storage_lens.metrics.StorageBytes.avg" + }, + "f13847d9-f404-41a8-8e17-12b47d683bd0X2": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of % Current Version Bytes", + "operationType": "math", + "params": { + "tinymathAst": { + "args": [ + "f13847d9-f404-41a8-8e17-12b47d683bd0X0", + "f13847d9-f404-41a8-8e17-12b47d683bd0X1" + ], + "location": { + "max": 117, + "min": 0 + }, + "name": "divide", + "text": "divide(sum(aws.s3_storage_lens.metrics.EncryptedStorageBytes.avg), sum(aws.s3_storage_lens.metrics.StorageBytes.avg))", + "type": "function" + } + }, + "references": [ + "f13847d9-f404-41a8-8e17-12b47d683bd0X0", + "f13847d9-f404-41a8-8e17-12b47d683bd0X1" + ], + "scale": "ratio" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "indexRefName": "filter-index-pattern-0", + "key": "data_stream.dataset", + "negate": false, + "params": { + "query": "aws.s3_storage_lens" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "data_stream.dataset": "aws.s3_storage_lens" + } + } + } + ], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "accessor": "f13847d9-f404-41a8-8e17-12b47d683bd0", + "layerId": "af6bd20f-099d-4817-a951-99bcba7e1752", + "layerType": "data" + } + }, + "title": "S3 Storage Lens Percentage Encrypted Bytes [AWS Metrics]", + "visualizationType": "lnsMetric" + }, + "coreMigrationVersion": "7.15.0", + "id": "aws-6445e130-41b5-11ec-a605-bff67d9b7872", + "migrationVersion": { + "lens": "7.15.0" + }, + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-af6bd20f-099d-4817-a951-99bcba7e1752", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "filter-index-pattern-0", + "type": "index-pattern" + } + ], + "type": "lens" +} \ No newline at end of file diff --git a/test/packages/aws/kibana/lens/aws-68f36fb0-f0a0-11eb-b61f-a53cb4913361.json b/test/packages/aws/kibana/lens/aws-68f36fb0-f0a0-11eb-b61f-a53cb4913361.json new file mode 100644 index 0000000000..6761ff6840 --- /dev/null +++ b/test/packages/aws/kibana/lens/aws-68f36fb0-f0a0-11eb-b61f-a53cb4913361.json @@ -0,0 +1,103 @@ +{ + "attributes": { + "description": null, + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "c6ed7acb-d119-41cc-99ce-cca114d1f1cb": { + "columnOrder": [ + "1bfe525f-e68d-4504-86bc-e80fb154192c", + "08fe8b96-3fe0-410f-8ee3-3ca1379bea49", + "08fe8b96-3fe0-410f-8ee3-3ca1379bea49X0" + ], + "columns": { + "08fe8b96-3fe0-410f-8ee3-3ca1379bea49": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Free Storage Bytes", + "operationType": "formula", + "params": { + "formula": "average(aws.rds.free_storage.bytes)", + "isFormulaBroken": false + }, + "references": [ + "08fe8b96-3fe0-410f-8ee3-3ca1379bea49X0" + ], + "scale": "ratio" + }, + "08fe8b96-3fe0-410f-8ee3-3ca1379bea49X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of average(aws.rds.free_storage.bytes)", + "operationType": "average", + "scale": "ratio", + "sourceField": "aws.rds.free_storage.bytes" + }, + "1bfe525f-e68d-4504-86bc-e80fb154192c": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of aws.rds.db_instance.identifier", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "fallback": true, + "type": "alphabetical" + }, + "orderDirection": "asc", + "otherBucket": true, + "size": 10 + }, + "scale": "ordinal", + "sourceField": "aws.rds.db_instance.identifier" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "columns": [ + { + "columnId": "08fe8b96-3fe0-410f-8ee3-3ca1379bea49", + "isTransposed": false + }, + { + "columnId": "1bfe525f-e68d-4504-86bc-e80fb154192c", + "isTransposed": false + } + ], + "layerId": "c6ed7acb-d119-41cc-99ce-cca114d1f1cb" + } + }, + "title": "RDS Free Storage Bytes [Metrics AWS]", + "visualizationType": "lnsDatatable" + }, + "coreMigrationVersion": "7.15.0", + "id": "aws-68f36fb0-f0a0-11eb-b61f-a53cb4913361", + "migrationVersion": { + "lens": "7.14.0" + }, + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-c6ed7acb-d119-41cc-99ce-cca114d1f1cb", + "type": "index-pattern" + } + ], + "type": "lens" +} \ No newline at end of file diff --git a/test/packages/aws/kibana/lens/aws-7543b3e0-41b5-11ec-a605-bff67d9b7872.json b/test/packages/aws/kibana/lens/aws-7543b3e0-41b5-11ec-a605-bff67d9b7872.json new file mode 100644 index 0000000000..c78fc66d03 --- /dev/null +++ b/test/packages/aws/kibana/lens/aws-7543b3e0-41b5-11ec-a605-bff67d9b7872.json @@ -0,0 +1,147 @@ +{ + "attributes": { + "description": "", + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "af6bd20f-099d-4817-a951-99bcba7e1752": { + "columnOrder": [ + "f13847d9-f404-41a8-8e17-12b47d683bd0", + "f13847d9-f404-41a8-8e17-12b47d683bd0X0", + "f13847d9-f404-41a8-8e17-12b47d683bd0X1", + "f13847d9-f404-41a8-8e17-12b47d683bd0X2" + ], + "columns": { + "f13847d9-f404-41a8-8e17-12b47d683bd0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "% Replicated Bytes", + "operationType": "formula", + "params": { + "format": { + "id": "percent", + "params": { + "decimals": 2 + } + }, + "formula": "divide(sum(aws.s3_storage_lens.metrics.ReplicatedStorageBytes.avg), sum(aws.s3_storage_lens.metrics.StorageBytes.avg))", + "isFormulaBroken": false + }, + "references": [ + "f13847d9-f404-41a8-8e17-12b47d683bd0X2" + ], + "scale": "ratio" + }, + "f13847d9-f404-41a8-8e17-12b47d683bd0X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of % Encrypted Bytes", + "operationType": "sum", + "scale": "ratio", + "sourceField": "aws.s3_storage_lens.metrics.ReplicatedStorageBytes.avg" + }, + "f13847d9-f404-41a8-8e17-12b47d683bd0X1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of % Encrypted Bytes", + "operationType": "sum", + "scale": "ratio", + "sourceField": "aws.s3_storage_lens.metrics.StorageBytes.avg" + }, + "f13847d9-f404-41a8-8e17-12b47d683bd0X2": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of % Encrypted Bytes", + "operationType": "math", + "params": { + "tinymathAst": { + "args": [ + "f13847d9-f404-41a8-8e17-12b47d683bd0X0", + "f13847d9-f404-41a8-8e17-12b47d683bd0X1" + ], + "location": { + "max": 118, + "min": 0 + }, + "name": "divide", + "text": "divide(sum(aws.s3_storage_lens.metrics.ReplicatedStorageBytes.avg), sum(aws.s3_storage_lens.metrics.StorageBytes.avg))", + "type": "function" + } + }, + "references": [ + "f13847d9-f404-41a8-8e17-12b47d683bd0X0", + "f13847d9-f404-41a8-8e17-12b47d683bd0X1" + ], + "scale": "ratio" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "indexRefName": "filter-index-pattern-0", + "key": "data_stream.dataset", + "negate": false, + "params": { + "query": "aws.s3_storage_lens" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "data_stream.dataset": "aws.s3_storage_lens" + } + } + } + ], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "accessor": "f13847d9-f404-41a8-8e17-12b47d683bd0", + "layerId": "af6bd20f-099d-4817-a951-99bcba7e1752", + "layerType": "data" + } + }, + "title": "S3 Storage Lens Percentage Replicated Bytes [AWS Metrics]", + "visualizationType": "lnsMetric" + }, + "coreMigrationVersion": "7.15.0", + "id": "aws-7543b3e0-41b5-11ec-a605-bff67d9b7872", + "migrationVersion": { + "lens": "7.15.0" + }, + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-af6bd20f-099d-4817-a951-99bcba7e1752", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "filter-index-pattern-0", + "type": "index-pattern" + } + ], + "type": "lens" +} \ No newline at end of file diff --git a/test/packages/aws/kibana/lens/aws-8560b400-f096-11eb-b61f-a53cb4913361.json b/test/packages/aws/kibana/lens/aws-8560b400-f096-11eb-b61f-a53cb4913361.json new file mode 100644 index 0000000000..fc00f03c91 --- /dev/null +++ b/test/packages/aws/kibana/lens/aws-8560b400-f096-11eb-b61f-a53cb4913361.json @@ -0,0 +1,145 @@ +{ + "attributes": { + "description": "", + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "75b24975-5ca3-4da5-bc1a-92013a901a21": { + "columnOrder": [ + "bfa06179-6b6f-43e5-a446-f856ff3e51bf", + "af96ca6c-7ab9-47a3-ad8b-29e1578c0076", + "6a87f496-b929-4d24-aede-325d54fedfa1", + "6a87f496-b929-4d24-aede-325d54fedfa1X0" + ], + "columns": { + "6a87f496-b929-4d24-aede-325d54fedfa1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Write Latency (Milliseconds)", + "operationType": "formula", + "params": { + "formula": "average(aws.rds.latency.write)", + "isFormulaBroken": false + }, + "references": [ + "6a87f496-b929-4d24-aede-325d54fedfa1X0" + ], + "scale": "ratio" + }, + "6a87f496-b929-4d24-aede-325d54fedfa1X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of average(aws.rds.latency.write)", + "operationType": "average", + "scale": "ratio", + "sourceField": "aws.rds.latency.write" + }, + "af96ca6c-7ab9-47a3-ad8b-29e1578c0076": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "bfa06179-6b6f-43e5-a446-f856ff3e51bf": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of aws.rds.db_instance.identifier", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "fallback": true, + "type": "alphabetical" + }, + "orderDirection": "asc", + "otherBucket": true, + "size": 10 + }, + "scale": "ordinal", + "sourceField": "aws.rds.db_instance.identifier" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "layers": [ + { + "accessors": [ + "6a87f496-b929-4d24-aede-325d54fedfa1" + ], + "layerId": "75b24975-5ca3-4da5-bc1a-92013a901a21", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "splitAccessor": "bfa06179-6b6f-43e5-a446-f856ff3e51bf", + "xAccessor": "af96ca6c-7ab9-47a3-ad8b-29e1578c0076" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide", + "yLeftExtent": { + "mode": "full" + }, + "yRightExtent": { + "mode": "full" + } + } + }, + "title": "RDS Write Latency in Milliseconds [Metrics AWS]", + "visualizationType": "lnsXY" + }, + "coreMigrationVersion": "7.15.0", + "id": "aws-8560b400-f096-11eb-b61f-a53cb4913361", + "migrationVersion": { + "lens": "7.14.0" + }, + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-75b24975-5ca3-4da5-bc1a-92013a901a21", + "type": "index-pattern" + } + ], + "type": "lens" +} \ No newline at end of file diff --git a/test/packages/aws/kibana/lens/aws-8614f2f0-41a2-11ec-a605-bff67d9b7872.json b/test/packages/aws/kibana/lens/aws-8614f2f0-41a2-11ec-a605-bff67d9b7872.json new file mode 100644 index 0000000000..05430d5bc4 --- /dev/null +++ b/test/packages/aws/kibana/lens/aws-8614f2f0-41a2-11ec-a605-bff67d9b7872.json @@ -0,0 +1,61 @@ +{ + "attributes": { + "description": "", + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "eab1960c-2b9a-4e4a-9380-c29e91cbb47f": { + "columnOrder": [ + "b7788a69-bb42-4dc4-b7a0-dfc01d99236f" + ], + "columns": { + "b7788a69-bb42-4dc4-b7a0-dfc01d99236f": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "All Requests", + "operationType": "sum", + "params": {}, + "scale": "ratio", + "sourceField": "aws.s3_storage_lens.metrics.AllRequests.avg" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "accessor": "b7788a69-bb42-4dc4-b7a0-dfc01d99236f", + "layerId": "eab1960c-2b9a-4e4a-9380-c29e91cbb47f", + "layerType": "data" + } + }, + "title": "S3 Storage Lens All Requests [Metrics AWS]", + "visualizationType": "lnsMetric" + }, + "coreMigrationVersion": "7.15.0", + "id": "aws-8614f2f0-41a2-11ec-a605-bff67d9b7872", + "migrationVersion": { + "lens": "7.15.0" + }, + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-eab1960c-2b9a-4e4a-9380-c29e91cbb47f", + "type": "index-pattern" + } + ], + "type": "lens" +} \ No newline at end of file diff --git a/test/packages/aws/kibana/lens/aws-8721fb80-41b5-11ec-a605-bff67d9b7872.json b/test/packages/aws/kibana/lens/aws-8721fb80-41b5-11ec-a605-bff67d9b7872.json new file mode 100644 index 0000000000..2e5ce41222 --- /dev/null +++ b/test/packages/aws/kibana/lens/aws-8721fb80-41b5-11ec-a605-bff67d9b7872.json @@ -0,0 +1,147 @@ +{ + "attributes": { + "description": "", + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "af6bd20f-099d-4817-a951-99bcba7e1752": { + "columnOrder": [ + "f13847d9-f404-41a8-8e17-12b47d683bd0", + "f13847d9-f404-41a8-8e17-12b47d683bd0X0", + "f13847d9-f404-41a8-8e17-12b47d683bd0X1", + "f13847d9-f404-41a8-8e17-12b47d683bd0X2" + ], + "columns": { + "f13847d9-f404-41a8-8e17-12b47d683bd0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "% Object Lock Bytes", + "operationType": "formula", + "params": { + "format": { + "id": "percent", + "params": { + "decimals": 2 + } + }, + "formula": "divide(sum(aws.s3_storage_lens.metrics.ObjectLockEnabledStorageBytes.avg), sum(aws.s3_storage_lens.metrics.StorageBytes.avg))", + "isFormulaBroken": false + }, + "references": [ + "f13847d9-f404-41a8-8e17-12b47d683bd0X2" + ], + "scale": "ratio" + }, + "f13847d9-f404-41a8-8e17-12b47d683bd0X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of % Encrypted Bytes", + "operationType": "sum", + "scale": "ratio", + "sourceField": "aws.s3_storage_lens.metrics.ObjectLockEnabledStorageBytes.avg" + }, + "f13847d9-f404-41a8-8e17-12b47d683bd0X1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of % Encrypted Bytes", + "operationType": "sum", + "scale": "ratio", + "sourceField": "aws.s3_storage_lens.metrics.StorageBytes.avg" + }, + "f13847d9-f404-41a8-8e17-12b47d683bd0X2": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of % Encrypted Bytes", + "operationType": "math", + "params": { + "tinymathAst": { + "args": [ + "f13847d9-f404-41a8-8e17-12b47d683bd0X0", + "f13847d9-f404-41a8-8e17-12b47d683bd0X1" + ], + "location": { + "max": 125, + "min": 0 + }, + "name": "divide", + "text": "divide(sum(aws.s3_storage_lens.metrics.ObjectLockEnabledStorageBytes.avg), sum(aws.s3_storage_lens.metrics.StorageBytes.avg))", + "type": "function" + } + }, + "references": [ + "f13847d9-f404-41a8-8e17-12b47d683bd0X0", + "f13847d9-f404-41a8-8e17-12b47d683bd0X1" + ], + "scale": "ratio" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "indexRefName": "filter-index-pattern-0", + "key": "data_stream.dataset", + "negate": false, + "params": { + "query": "aws.s3_storage_lens" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "data_stream.dataset": "aws.s3_storage_lens" + } + } + } + ], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "accessor": "f13847d9-f404-41a8-8e17-12b47d683bd0", + "layerId": "af6bd20f-099d-4817-a951-99bcba7e1752", + "layerType": "data" + } + }, + "title": "S3 Storage Lens Percentage Object Lock Bytes [AWS Metrics]", + "visualizationType": "lnsMetric" + }, + "coreMigrationVersion": "7.15.0", + "id": "aws-8721fb80-41b5-11ec-a605-bff67d9b7872", + "migrationVersion": { + "lens": "7.15.0" + }, + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-af6bd20f-099d-4817-a951-99bcba7e1752", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "filter-index-pattern-0", + "type": "index-pattern" + } + ], + "type": "lens" +} \ No newline at end of file diff --git a/test/packages/aws/kibana/lens/aws-8d42b050-41a5-11ec-a605-bff67d9b7872.json b/test/packages/aws/kibana/lens/aws-8d42b050-41a5-11ec-a605-bff67d9b7872.json new file mode 100644 index 0000000000..d632ba45f5 --- /dev/null +++ b/test/packages/aws/kibana/lens/aws-8d42b050-41a5-11ec-a605-bff67d9b7872.json @@ -0,0 +1,147 @@ +{ + "attributes": { + "description": "", + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "af6bd20f-099d-4817-a951-99bcba7e1752": { + "columnOrder": [ + "f13847d9-f404-41a8-8e17-12b47d683bd0", + "f13847d9-f404-41a8-8e17-12b47d683bd0X0", + "f13847d9-f404-41a8-8e17-12b47d683bd0X1", + "f13847d9-f404-41a8-8e17-12b47d683bd0X2" + ], + "columns": { + "f13847d9-f404-41a8-8e17-12b47d683bd0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Average Object Size", + "operationType": "formula", + "params": { + "format": { + "id": "bytes", + "params": { + "decimals": 2 + } + }, + "formula": "divide(sum(aws.s3_storage_lens.metrics.StorageBytes.avg), sum(aws.s3_storage_lens.metrics.ObjectCount.avg))", + "isFormulaBroken": false + }, + "references": [ + "f13847d9-f404-41a8-8e17-12b47d683bd0X2" + ], + "scale": "ratio" + }, + "f13847d9-f404-41a8-8e17-12b47d683bd0X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Average Object Size", + "operationType": "sum", + "scale": "ratio", + "sourceField": "aws.s3_storage_lens.metrics.StorageBytes.avg" + }, + "f13847d9-f404-41a8-8e17-12b47d683bd0X1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Average Object Size", + "operationType": "sum", + "scale": "ratio", + "sourceField": "aws.s3_storage_lens.metrics.ObjectCount.avg" + }, + "f13847d9-f404-41a8-8e17-12b47d683bd0X2": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Average Object Size", + "operationType": "math", + "params": { + "tinymathAst": { + "args": [ + "f13847d9-f404-41a8-8e17-12b47d683bd0X0", + "f13847d9-f404-41a8-8e17-12b47d683bd0X1" + ], + "location": { + "max": 107, + "min": 0 + }, + "name": "divide", + "text": "divide(sum(aws.s3_storage_lens.metrics.StorageBytes.avg), sum(aws.s3_storage_lens.metrics.ObjectCount.avg))", + "type": "function" + } + }, + "references": [ + "f13847d9-f404-41a8-8e17-12b47d683bd0X0", + "f13847d9-f404-41a8-8e17-12b47d683bd0X1" + ], + "scale": "ratio" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "indexRefName": "filter-index-pattern-0", + "key": "data_stream.dataset", + "negate": false, + "params": { + "query": "aws.s3_storage_lens" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "data_stream.dataset": "aws.s3_storage_lens" + } + } + } + ], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "accessor": "f13847d9-f404-41a8-8e17-12b47d683bd0", + "layerId": "af6bd20f-099d-4817-a951-99bcba7e1752", + "layerType": "data" + } + }, + "title": "Average Object Size [AWS Metrics]", + "visualizationType": "lnsMetric" + }, + "coreMigrationVersion": "7.15.0", + "id": "aws-8d42b050-41a5-11ec-a605-bff67d9b7872", + "migrationVersion": { + "lens": "7.15.0" + }, + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-af6bd20f-099d-4817-a951-99bcba7e1752", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "filter-index-pattern-0", + "type": "index-pattern" + } + ], + "type": "lens" +} \ No newline at end of file diff --git a/test/packages/aws/kibana/lens/aws-955b2000-41a1-11ec-a605-bff67d9b7872.json b/test/packages/aws/kibana/lens/aws-955b2000-41a1-11ec-a605-bff67d9b7872.json new file mode 100644 index 0000000000..4e4a36e680 --- /dev/null +++ b/test/packages/aws/kibana/lens/aws-955b2000-41a1-11ec-a605-bff67d9b7872.json @@ -0,0 +1,86 @@ +{ + "attributes": { + "description": "", + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "eab1960c-2b9a-4e4a-9380-c29e91cbb47f": { + "columnOrder": [ + "a80c40e7-af1b-4ac9-ba2e-baa7c13a729a", + "a80c40e7-af1b-4ac9-ba2e-baa7c13a729aX0" + ], + "columns": { + "a80c40e7-af1b-4ac9-ba2e-baa7c13a729a": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Total Storage", + "operationType": "formula", + "params": { + "format": { + "id": "bytes", + "params": { + "decimals": 2 + } + }, + "formula": "sum(aws.s3_storage_lens.metrics.StorageBytes.avg, kql='aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" ')", + "isFormulaBroken": false + }, + "references": [ + "a80c40e7-af1b-4ac9-ba2e-baa7c13a729aX0" + ], + "scale": "ratio" + }, + "a80c40e7-af1b-4ac9-ba2e-baa7c13a729aX0": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" " + }, + "isBucketed": false, + "label": "Part of sum(aws.s3_storage_lens.metrics.StorageBytes.avg, kql='aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" ')", + "operationType": "sum", + "scale": "ratio", + "sourceField": "aws.s3_storage_lens.metrics.StorageBytes.avg" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "accessor": "a80c40e7-af1b-4ac9-ba2e-baa7c13a729a", + "layerId": "eab1960c-2b9a-4e4a-9380-c29e91cbb47f", + "layerType": "data" + } + }, + "title": "S3 Storage Lens Total Storage [Metrics AWS]", + "visualizationType": "lnsMetric" + }, + "coreMigrationVersion": "7.15.0", + "id": "aws-955b2000-41a1-11ec-a605-bff67d9b7872", + "migrationVersion": { + "lens": "7.15.0" + }, + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-eab1960c-2b9a-4e4a-9380-c29e91cbb47f", + "type": "index-pattern" + } + ], + "type": "lens" +} \ No newline at end of file diff --git a/test/packages/aws/kibana/lens/aws-9a64ca30-427b-11ec-9b39-b9df88395505.json b/test/packages/aws/kibana/lens/aws-9a64ca30-427b-11ec-9b39-b9df88395505.json new file mode 100644 index 0000000000..3c45f40f6f --- /dev/null +++ b/test/packages/aws/kibana/lens/aws-9a64ca30-427b-11ec-9b39-b9df88395505.json @@ -0,0 +1,300 @@ +{ + "attributes": { + "description": "", + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "93f6aedf-d508-4bd9-9cf0-f985b7aae3cd": { + "columnOrder": [ + "11a794cc-93d0-4b87-be6e-da1a9ccb12b1", + "658be95f-9664-4a08-88c6-ec19d3558fdf", + "690af176-5839-42ba-8441-e4db17bbe523", + "11a794cc-93d0-4b87-be6e-da1a9ccb12b1X0", + "11a794cc-93d0-4b87-be6e-da1a9ccb12b1X1", + "11a794cc-93d0-4b87-be6e-da1a9ccb12b1X2", + "658be95f-9664-4a08-88c6-ec19d3558fdfX0", + "658be95f-9664-4a08-88c6-ec19d3558fdfX1", + "658be95f-9664-4a08-88c6-ec19d3558fdfX2", + "690af176-5839-42ba-8441-e4db17bbe523X0", + "690af176-5839-42ba-8441-e4db17bbe523X1", + "690af176-5839-42ba-8441-e4db17bbe523X2" + ], + "columns": { + "11a794cc-93d0-4b87-be6e-da1a9ccb12b1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "% Object Lock Bytes", + "operationType": "formula", + "params": { + "format": { + "id": "percent", + "params": { + "decimals": 2 + } + }, + "formula": "divide(sum(aws.s3_storage_lens.metrics.ObjectLockEnabledStorageBytes.avg, kql='aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" '), sum(aws.s3_storage_lens.metrics.StorageBytes.avg, kql='aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" '))", + "isFormulaBroken": false + }, + "references": [ + "11a794cc-93d0-4b87-be6e-da1a9ccb12b1X2" + ], + "scale": "ratio" + }, + "11a794cc-93d0-4b87-be6e-da1a9ccb12b1X0": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" " + }, + "isBucketed": false, + "label": "Part of % Object Lock Bytes", + "operationType": "sum", + "scale": "ratio", + "sourceField": "aws.s3_storage_lens.metrics.ObjectLockEnabledStorageBytes.avg" + }, + "11a794cc-93d0-4b87-be6e-da1a9ccb12b1X1": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" " + }, + "isBucketed": false, + "label": "Part of % Object Lock Bytes", + "operationType": "sum", + "scale": "ratio", + "sourceField": "aws.s3_storage_lens.metrics.StorageBytes.avg" + }, + "11a794cc-93d0-4b87-be6e-da1a9ccb12b1X2": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of % Object Lock Bytes", + "operationType": "math", + "params": { + "tinymathAst": { + "args": [ + "11a794cc-93d0-4b87-be6e-da1a9ccb12b1X0", + "11a794cc-93d0-4b87-be6e-da1a9ccb12b1X1" + ], + "location": { + "max": 305, + "min": 0 + }, + "name": "divide", + "text": "divide(sum(aws.s3_storage_lens.metrics.ObjectLockEnabledStorageBytes.avg, kql='aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" '), sum(aws.s3_storage_lens.metrics.StorageBytes.avg, kql='aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" '))", + "type": "function" + } + }, + "references": [ + "11a794cc-93d0-4b87-be6e-da1a9ccb12b1X0", + "11a794cc-93d0-4b87-be6e-da1a9ccb12b1X1" + ], + "scale": "ratio" + }, + "658be95f-9664-4a08-88c6-ec19d3558fdf": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "% Replicated Bytes", + "operationType": "formula", + "params": { + "format": { + "id": "percent", + "params": { + "decimals": 2 + } + }, + "formula": "divide(sum(aws.s3_storage_lens.metrics.ReplicatedStorageBytes.avg, kql='aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" '), sum(aws.s3_storage_lens.metrics.StorageBytes.avg, kql='aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" '))", + "isFormulaBroken": false + }, + "references": [ + "658be95f-9664-4a08-88c6-ec19d3558fdfX2" + ], + "scale": "ratio" + }, + "658be95f-9664-4a08-88c6-ec19d3558fdfX0": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" " + }, + "isBucketed": false, + "label": "Part of % Replicated Bytes", + "operationType": "sum", + "scale": "ratio", + "sourceField": "aws.s3_storage_lens.metrics.ReplicatedStorageBytes.avg" + }, + "658be95f-9664-4a08-88c6-ec19d3558fdfX1": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" " + }, + "isBucketed": false, + "label": "Part of % Replicated Bytes", + "operationType": "sum", + "scale": "ratio", + "sourceField": "aws.s3_storage_lens.metrics.StorageBytes.avg" + }, + "658be95f-9664-4a08-88c6-ec19d3558fdfX2": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of % Replicated Bytes", + "operationType": "math", + "params": { + "tinymathAst": { + "args": [ + "658be95f-9664-4a08-88c6-ec19d3558fdfX0", + "658be95f-9664-4a08-88c6-ec19d3558fdfX1" + ], + "location": { + "max": 298, + "min": 0 + }, + "name": "divide", + "text": "divide(sum(aws.s3_storage_lens.metrics.ReplicatedStorageBytes.avg, kql='aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" '), sum(aws.s3_storage_lens.metrics.StorageBytes.avg, kql='aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" '))", + "type": "function" + } + }, + "references": [ + "658be95f-9664-4a08-88c6-ec19d3558fdfX0", + "658be95f-9664-4a08-88c6-ec19d3558fdfX1" + ], + "scale": "ratio" + }, + "690af176-5839-42ba-8441-e4db17bbe523": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "% Encrypted Bytes", + "operationType": "formula", + "params": { + "format": { + "id": "percent", + "params": { + "decimals": 2 + } + }, + "formula": "divide(sum(aws.s3_storage_lens.metrics.EncryptedStorageBytes.avg, kql='aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" '), sum(aws.s3_storage_lens.metrics.StorageBytes.avg, kql='aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" '))", + "isFormulaBroken": false + }, + "references": [ + "690af176-5839-42ba-8441-e4db17bbe523X2" + ], + "scale": "ratio" + }, + "690af176-5839-42ba-8441-e4db17bbe523X0": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" " + }, + "isBucketed": false, + "label": "Part of % Encrypted Bytes", + "operationType": "sum", + "scale": "ratio", + "sourceField": "aws.s3_storage_lens.metrics.EncryptedStorageBytes.avg" + }, + "690af176-5839-42ba-8441-e4db17bbe523X1": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" " + }, + "isBucketed": false, + "label": "Part of % Encrypted Bytes", + "operationType": "sum", + "scale": "ratio", + "sourceField": "aws.s3_storage_lens.metrics.StorageBytes.avg" + }, + "690af176-5839-42ba-8441-e4db17bbe523X2": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of % Encrypted Bytes", + "operationType": "math", + "params": { + "tinymathAst": { + "args": [ + "690af176-5839-42ba-8441-e4db17bbe523X0", + "690af176-5839-42ba-8441-e4db17bbe523X1" + ], + "location": { + "max": 297, + "min": 0 + }, + "name": "divide", + "text": "divide(sum(aws.s3_storage_lens.metrics.EncryptedStorageBytes.avg, kql='aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" '), sum(aws.s3_storage_lens.metrics.StorageBytes.avg, kql='aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" '))", + "type": "function" + } + }, + "references": [ + "690af176-5839-42ba-8441-e4db17bbe523X0", + "690af176-5839-42ba-8441-e4db17bbe523X1" + ], + "scale": "ratio" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "columns": [ + { + "alignment": "center", + "columnId": "11a794cc-93d0-4b87-be6e-da1a9ccb12b1", + "isTransposed": false + }, + { + "alignment": "center", + "columnId": "658be95f-9664-4a08-88c6-ec19d3558fdf", + "isTransposed": false + }, + { + "alignment": "center", + "columnId": "690af176-5839-42ba-8441-e4db17bbe523", + "isTransposed": false + } + ], + "layerId": "93f6aedf-d508-4bd9-9cf0-f985b7aae3cd", + "layerType": "data" + } + }, + "title": "S3 Storage Lens Data Protection [AWS Metrics]", + "visualizationType": "lnsDatatable" + }, + "coreMigrationVersion": "7.15.0", + "id": "aws-9a64ca30-427b-11ec-9b39-b9df88395505", + "migrationVersion": { + "lens": "7.15.0" + }, + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-93f6aedf-d508-4bd9-9cf0-f985b7aae3cd", + "type": "index-pattern" + } + ], + "type": "lens" +} \ No newline at end of file diff --git a/test/packages/aws/kibana/lens/aws-9fb44b60-41b2-11ec-a605-bff67d9b7872.json b/test/packages/aws/kibana/lens/aws-9fb44b60-41b2-11ec-a605-bff67d9b7872.json new file mode 100644 index 0000000000..e6a7e2e0e7 --- /dev/null +++ b/test/packages/aws/kibana/lens/aws-9fb44b60-41b2-11ec-a605-bff67d9b7872.json @@ -0,0 +1,208 @@ +{ + "attributes": { + "description": "", + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "af6bd20f-099d-4817-a951-99bcba7e1752": { + "columnOrder": [ + "6af18f84-7f98-4aec-88d2-51eb0e177fa7", + "f13847d9-f404-41a8-8e17-12b47d683bd0", + "f13847d9-f404-41a8-8e17-12b47d683bd0X0", + "f13847d9-f404-41a8-8e17-12b47d683bd0X1", + "f13847d9-f404-41a8-8e17-12b47d683bd0X2" + ], + "columns": { + "6af18f84-7f98-4aec-88d2-51eb0e177fa7": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "interval": "1d" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "f13847d9-f404-41a8-8e17-12b47d683bd0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "% Current Version Bytes", + "operationType": "formula", + "params": { + "format": { + "id": "percent", + "params": { + "decimals": 2 + } + }, + "formula": "divide(sum(aws.s3_storage_lens.metrics.CurrentVersionStorageBytes.avg, kql='aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" '), sum(aws.s3_storage_lens.metrics.StorageBytes.avg, kql='aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" '))", + "isFormulaBroken": false + }, + "references": [ + "f13847d9-f404-41a8-8e17-12b47d683bd0X2" + ], + "scale": "ratio" + }, + "f13847d9-f404-41a8-8e17-12b47d683bd0X0": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" " + }, + "isBucketed": false, + "label": "Part of % Current Version Bytes", + "operationType": "sum", + "scale": "ratio", + "sourceField": "aws.s3_storage_lens.metrics.CurrentVersionStorageBytes.avg" + }, + "f13847d9-f404-41a8-8e17-12b47d683bd0X1": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" " + }, + "isBucketed": false, + "label": "Part of % Current Version Bytes", + "operationType": "sum", + "scale": "ratio", + "sourceField": "aws.s3_storage_lens.metrics.StorageBytes.avg" + }, + "f13847d9-f404-41a8-8e17-12b47d683bd0X2": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of % Current Version Bytes", + "operationType": "math", + "params": { + "tinymathAst": { + "args": [ + "f13847d9-f404-41a8-8e17-12b47d683bd0X0", + "f13847d9-f404-41a8-8e17-12b47d683bd0X1" + ], + "location": { + "max": 302, + "min": 0 + }, + "name": "divide", + "text": "divide(sum(aws.s3_storage_lens.metrics.CurrentVersionStorageBytes.avg, kql='aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" '), sum(aws.s3_storage_lens.metrics.StorageBytes.avg, kql='aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" '))", + "type": "function" + } + }, + "references": [ + "f13847d9-f404-41a8-8e17-12b47d683bd0X0", + "f13847d9-f404-41a8-8e17-12b47d683bd0X1" + ], + "scale": "ratio" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "indexRefName": "filter-index-pattern-0", + "key": "data_stream.dataset", + "negate": false, + "params": { + "query": "aws.s3_storage_lens" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "data_stream.dataset": "aws.s3_storage_lens" + } + } + } + ], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "f13847d9-f404-41a8-8e17-12b47d683bd0" + ], + "layerId": "af6bd20f-099d-4817-a951-99bcba7e1752", + "layerType": "data", + "seriesType": "line", + "xAccessor": "6af18f84-7f98-4aec-88d2-51eb0e177fa7" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide", + "yLeftExtent": { + "mode": "full" + }, + "yRightExtent": { + "mode": "full" + } + } + }, + "title": "S3 Storage Lens Percentage Current Version Bytes [AWS Metrics]", + "visualizationType": "lnsXY" + }, + "coreMigrationVersion": "7.15.0", + "id": "aws-9fb44b60-41b2-11ec-a605-bff67d9b7872", + "migrationVersion": { + "lens": "7.15.0" + }, + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-af6bd20f-099d-4817-a951-99bcba7e1752", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "filter-index-pattern-0", + "type": "index-pattern" + } + ], + "type": "lens" +} \ No newline at end of file diff --git a/test/packages/aws/kibana/lens/aws-b0077d10-f09b-11eb-b61f-a53cb4913361.json b/test/packages/aws/kibana/lens/aws-b0077d10-f09b-11eb-b61f-a53cb4913361.json new file mode 100644 index 0000000000..8564e41dc1 --- /dev/null +++ b/test/packages/aws/kibana/lens/aws-b0077d10-f09b-11eb-b61f-a53cb4913361.json @@ -0,0 +1,129 @@ +{ + "attributes": { + "description": null, + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "b8d09be0-e20a-4f42-b08e-1da4c3cc8efd": { + "columnOrder": [ + "e676afd3-ebd5-434a-85d7-a1a708b9a32f", + "545fe110-3ab4-4a3d-99d9-9eae69d6ff07", + "8293fc99-aed6-44a3-83ee-a498d2200a46" + ], + "columns": { + "545fe110-3ab4-4a3d-99d9-9eae69d6ff07": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of aws.rds.db_instance.identifier", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "columnId": "8293fc99-aed6-44a3-83ee-a498d2200a46", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "size": 10 + }, + "scale": "ordinal", + "sourceField": "aws.rds.db_instance.identifier" + }, + "8293fc99-aed6-44a3-83ee-a498d2200a46": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Insert Throughput Count/Second", + "operationType": "average", + "scale": "ratio", + "sourceField": "aws.rds.throughput.insert" + }, + "e676afd3-ebd5-434a-85d7-a1a708b9a32f": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "layers": [ + { + "accessors": [ + "8293fc99-aed6-44a3-83ee-a498d2200a46" + ], + "layerId": "b8d09be0-e20a-4f42-b08e-1da4c3cc8efd", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "splitAccessor": "545fe110-3ab4-4a3d-99d9-9eae69d6ff07", + "xAccessor": "e676afd3-ebd5-434a-85d7-a1a708b9a32f" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide", + "yLeftExtent": { + "mode": "full" + }, + "yRightExtent": { + "mode": "full" + } + } + }, + "title": "RDS Insert Throughput [Metrics AWS]", + "visualizationType": "lnsXY" + }, + "coreMigrationVersion": "7.15.0", + "id": "aws-b0077d10-f09b-11eb-b61f-a53cb4913361", + "migrationVersion": { + "lens": "7.14.0" + }, + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-b8d09be0-e20a-4f42-b08e-1da4c3cc8efd", + "type": "index-pattern" + } + ], + "type": "lens" +} \ No newline at end of file diff --git a/test/packages/aws/kibana/lens/aws-bbc80900-f09e-11eb-b61f-a53cb4913361.json b/test/packages/aws/kibana/lens/aws-bbc80900-f09e-11eb-b61f-a53cb4913361.json new file mode 100644 index 0000000000..8ef68d9724 --- /dev/null +++ b/test/packages/aws/kibana/lens/aws-bbc80900-f09e-11eb-b61f-a53cb4913361.json @@ -0,0 +1,145 @@ +{ + "attributes": { + "description": "", + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "14d4ba6b-f4e1-4d40-818a-6aa829d90422": { + "columnOrder": [ + "8954842c-4056-46ef-adfc-29dfc3b0cbd3", + "40493df1-c805-49eb-8dfa-9ff81f7acd4b", + "c7c3ebb2-d611-40a5-aab3-491fa36fe729", + "c7c3ebb2-d611-40a5-aab3-491fa36fe729X0" + ], + "columns": { + "40493df1-c805-49eb-8dfa-9ff81f7acd4b": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "8954842c-4056-46ef-adfc-29dfc3b0cbd3": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of aws.rds.db_instance.identifier", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "fallback": true, + "type": "alphabetical" + }, + "orderDirection": "asc", + "otherBucket": true, + "size": 10 + }, + "scale": "ordinal", + "sourceField": "aws.rds.db_instance.identifier" + }, + "c7c3ebb2-d611-40a5-aab3-491fa36fe729": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Queue Depth (Count)", + "operationType": "formula", + "params": { + "formula": "average(aws.rds.disk_queue_depth)", + "isFormulaBroken": false + }, + "references": [ + "c7c3ebb2-d611-40a5-aab3-491fa36fe729X0" + ], + "scale": "ratio" + }, + "c7c3ebb2-d611-40a5-aab3-491fa36fe729X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of Queue Depth (Count)", + "operationType": "average", + "scale": "ratio", + "sourceField": "aws.rds.disk_queue_depth" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "layers": [ + { + "accessors": [ + "c7c3ebb2-d611-40a5-aab3-491fa36fe729" + ], + "layerId": "14d4ba6b-f4e1-4d40-818a-6aa829d90422", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "splitAccessor": "8954842c-4056-46ef-adfc-29dfc3b0cbd3", + "xAccessor": "40493df1-c805-49eb-8dfa-9ff81f7acd4b" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide", + "yLeftExtent": { + "mode": "full" + }, + "yRightExtent": { + "mode": "full" + } + } + }, + "title": "RDS Disk Queue Depth [Metrics AWS]", + "visualizationType": "lnsXY" + }, + "coreMigrationVersion": "7.15.0", + "id": "aws-bbc80900-f09e-11eb-b61f-a53cb4913361", + "migrationVersion": { + "lens": "7.14.0" + }, + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-14d4ba6b-f4e1-4d40-818a-6aa829d90422", + "type": "index-pattern" + } + ], + "type": "lens" +} \ No newline at end of file diff --git a/test/packages/aws/kibana/lens/aws-c4419a90-f091-11eb-b61f-a53cb4913361.json b/test/packages/aws/kibana/lens/aws-c4419a90-f091-11eb-b61f-a53cb4913361.json new file mode 100644 index 0000000000..f6c53f2ed4 --- /dev/null +++ b/test/packages/aws/kibana/lens/aws-c4419a90-f091-11eb-b61f-a53cb4913361.json @@ -0,0 +1,145 @@ +{ + "attributes": { + "description": null, + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "dd0a4706-5286-4976-9bc4-f5e7a4964bf6": { + "columnOrder": [ + "a2bb9c7a-0ddc-4bf7-ae24-98a535a916cc", + "103900c0-dcfa-416f-a272-6efa09c84fce", + "18e6079e-e955-41d0-8196-d2b932cf1fa6", + "18e6079e-e955-41d0-8196-d2b932cf1fa6X0" + ], + "columns": { + "103900c0-dcfa-416f-a272-6efa09c84fce": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "18e6079e-e955-41d0-8196-d2b932cf1fa6": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": " average(aws.rds.cpu.total.pct)", + "operationType": "formula", + "params": { + "formula": "average(aws.rds.cpu.total.pct)", + "isFormulaBroken": false + }, + "references": [ + "18e6079e-e955-41d0-8196-d2b932cf1fa6X0" + ], + "scale": "ratio" + }, + "18e6079e-e955-41d0-8196-d2b932cf1fa6X0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of ", + "operationType": "average", + "scale": "ratio", + "sourceField": "aws.rds.cpu.total.pct" + }, + "a2bb9c7a-0ddc-4bf7-ae24-98a535a916cc": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of aws.rds.db_instance.identifier", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "fallback": true, + "type": "alphabetical" + }, + "orderDirection": "asc", + "otherBucket": true, + "size": 10 + }, + "scale": "ordinal", + "sourceField": "aws.rds.db_instance.identifier" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "layers": [ + { + "accessors": [ + "18e6079e-e955-41d0-8196-d2b932cf1fa6" + ], + "layerId": "dd0a4706-5286-4976-9bc4-f5e7a4964bf6", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "splitAccessor": "a2bb9c7a-0ddc-4bf7-ae24-98a535a916cc", + "xAccessor": "103900c0-dcfa-416f-a272-6efa09c84fce" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide", + "yLeftExtent": { + "mode": "full" + }, + "yRightExtent": { + "mode": "full" + } + } + }, + "title": "RDS CPU Total Pct [Metrics AWS]", + "visualizationType": "lnsXY" + }, + "coreMigrationVersion": "7.15.0", + "id": "aws-c4419a90-f091-11eb-b61f-a53cb4913361", + "migrationVersion": { + "lens": "7.14.0" + }, + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-dd0a4706-5286-4976-9bc4-f5e7a4964bf6", + "type": "index-pattern" + } + ], + "type": "lens" +} \ No newline at end of file diff --git a/test/packages/aws/kibana/lens/aws-cd0300c0-41a8-11ec-a605-bff67d9b7872.json b/test/packages/aws/kibana/lens/aws-cd0300c0-41a8-11ec-a605-bff67d9b7872.json new file mode 100644 index 0000000000..ef70a84946 --- /dev/null +++ b/test/packages/aws/kibana/lens/aws-cd0300c0-41a8-11ec-a605-bff67d9b7872.json @@ -0,0 +1,216 @@ +{ + "attributes": { + "description": "", + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "eab1960c-2b9a-4e4a-9380-c29e91cbb47f": { + "columnOrder": [ + "84cbc05d-02f3-4bcd-9753-e1e0ef7d2fd9", + "a80c40e7-af1b-4ac9-ba2e-baa7c13a729a", + "f396053a-8fda-4ccb-88c5-b3ea34207c79", + "a80c40e7-af1b-4ac9-ba2e-baa7c13a729aX0", + "f396053a-8fda-4ccb-88c5-b3ea34207c79X0" + ], + "columns": { + "84cbc05d-02f3-4bcd-9753-e1e0ef7d2fd9": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Storage Classes", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "fallback": true, + "type": "alphabetical" + }, + "orderDirection": "asc", + "otherBucket": true, + "size": 5 + }, + "scale": "ordinal", + "sourceField": "aws.dimensions.storage_class" + }, + "a80c40e7-af1b-4ac9-ba2e-baa7c13a729a": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Total Storage", + "operationType": "formula", + "params": { + "format": { + "id": "bytes", + "params": { + "decimals": 2 + } + }, + "formula": "sum(aws.s3_storage_lens.metrics.StorageBytes.avg, kql='aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" ')", + "isFormulaBroken": false + }, + "references": [ + "a80c40e7-af1b-4ac9-ba2e-baa7c13a729aX0" + ], + "scale": "ratio" + }, + "a80c40e7-af1b-4ac9-ba2e-baa7c13a729aX0": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" " + }, + "isBucketed": false, + "label": "Part of Total Storage", + "operationType": "sum", + "scale": "ratio", + "sourceField": "aws.s3_storage_lens.metrics.StorageBytes.avg" + }, + "f396053a-8fda-4ccb-88c5-b3ea34207c79": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Object Count", + "operationType": "formula", + "params": { + "formula": "sum(aws.s3_storage_lens.metrics.ObjectCount.avg, kql='aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" ')", + "isFormulaBroken": false + }, + "references": [ + "f396053a-8fda-4ccb-88c5-b3ea34207c79X0" + ], + "scale": "ratio" + }, + "f396053a-8fda-4ccb-88c5-b3ea34207c79X0": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" " + }, + "isBucketed": false, + "label": "Part of Object count", + "operationType": "sum", + "scale": "ratio", + "sourceField": "aws.s3_storage_lens.metrics.ObjectCount.avg" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "indexRefName": "filter-index-pattern-0", + "key": "data_stream.dataset", + "negate": false, + "params": { + "query": "aws.s3_storage_lens" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "data_stream.dataset": "aws.s3_storage_lens" + } + } + } + ], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "a80c40e7-af1b-4ac9-ba2e-baa7c13a729a", + "f396053a-8fda-4ccb-88c5-b3ea34207c79" + ], + "layerId": "eab1960c-2b9a-4e4a-9380-c29e91cbb47f", + "layerType": "data", + "seriesType": "bar", + "xAccessor": "84cbc05d-02f3-4bcd-9753-e1e0ef7d2fd9", + "yConfig": [ + { + "axisMode": "left", + "color": "#e7664c", + "forAccessor": "a80c40e7-af1b-4ac9-ba2e-baa7c13a729a" + }, + { + "axisMode": "right", + "forAccessor": "f396053a-8fda-4ccb-88c5-b3ea34207c79" + } + ] + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "bar", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide", + "yLeftExtent": { + "mode": "full" + }, + "yRightExtent": { + "mode": "full" + } + } + }, + "title": "S3 Storage Lens Storage Class Distribution [Metrics AWS]", + "visualizationType": "lnsXY" + }, + "coreMigrationVersion": "7.15.0", + "id": "aws-cd0300c0-41a8-11ec-a605-bff67d9b7872", + "migrationVersion": { + "lens": "7.15.0" + }, + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-eab1960c-2b9a-4e4a-9380-c29e91cbb47f", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "filter-index-pattern-0", + "type": "index-pattern" + } + ], + "type": "lens" +} \ No newline at end of file diff --git a/test/packages/aws/kibana/lens/aws-d33fa860-41a0-11ec-a605-bff67d9b7872.json b/test/packages/aws/kibana/lens/aws-d33fa860-41a0-11ec-a605-bff67d9b7872.json new file mode 100644 index 0000000000..c73762470b --- /dev/null +++ b/test/packages/aws/kibana/lens/aws-d33fa860-41a0-11ec-a605-bff67d9b7872.json @@ -0,0 +1,80 @@ +{ + "attributes": { + "description": "", + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "eab1960c-2b9a-4e4a-9380-c29e91cbb47f": { + "columnOrder": [ + "e7f0d2ee-3ff1-463f-9b94-c9f93296d0e0", + "e7f0d2ee-3ff1-463f-9b94-c9f93296d0e0X0" + ], + "columns": { + "e7f0d2ee-3ff1-463f-9b94-c9f93296d0e0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Object Count", + "operationType": "formula", + "params": { + "formula": "sum(aws.s3_storage_lens.metrics.ObjectCount.avg, kql='aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" ')", + "isFormulaBroken": false + }, + "references": [ + "e7f0d2ee-3ff1-463f-9b94-c9f93296d0e0X0" + ], + "scale": "ratio" + }, + "e7f0d2ee-3ff1-463f-9b94-c9f93296d0e0X0": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" " + }, + "isBucketed": false, + "label": "Part of Object Count", + "operationType": "sum", + "scale": "ratio", + "sourceField": "aws.s3_storage_lens.metrics.ObjectCount.avg" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "accessor": "e7f0d2ee-3ff1-463f-9b94-c9f93296d0e0", + "layerId": "eab1960c-2b9a-4e4a-9380-c29e91cbb47f", + "layerType": "data" + } + }, + "title": "S3 Storage Lens Object Count [Metrics AWS]", + "visualizationType": "lnsMetric" + }, + "coreMigrationVersion": "7.15.0", + "id": "aws-d33fa860-41a0-11ec-a605-bff67d9b7872", + "migrationVersion": { + "lens": "7.15.0" + }, + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-eab1960c-2b9a-4e4a-9380-c29e91cbb47f", + "type": "index-pattern" + } + ], + "type": "lens" +} \ No newline at end of file diff --git a/test/packages/aws/kibana/lens/aws-ed494970-41b2-11ec-a605-bff67d9b7872.json b/test/packages/aws/kibana/lens/aws-ed494970-41b2-11ec-a605-bff67d9b7872.json new file mode 100644 index 0000000000..dde566ca0a --- /dev/null +++ b/test/packages/aws/kibana/lens/aws-ed494970-41b2-11ec-a605-bff67d9b7872.json @@ -0,0 +1,107 @@ +{ + "attributes": { + "description": "", + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "af6bd20f-099d-4817-a951-99bcba7e1752": { + "columnOrder": [ + "893e6b97-057d-4ff1-92b0-03236debf90e", + "893e6b97-057d-4ff1-92b0-03236debf90eX0" + ], + "columns": { + "893e6b97-057d-4ff1-92b0-03236debf90e": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Delete Marker Object Count", + "operationType": "formula", + "params": { + "formula": "sum(aws.s3_storage_lens.metrics.DeleteMarkerObjectCount.avg, kql='aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" ')", + "isFormulaBroken": false + }, + "references": [ + "893e6b97-057d-4ff1-92b0-03236debf90eX0" + ], + "scale": "ratio" + }, + "893e6b97-057d-4ff1-92b0-03236debf90eX0": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" " + }, + "isBucketed": false, + "label": "Part of sum(aws.s3_storage_lens.metrics.DeleteMarkerObjectCount.avg, kql='aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" ')", + "operationType": "sum", + "scale": "ratio", + "sourceField": "aws.s3_storage_lens.metrics.DeleteMarkerObjectCount.avg" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "indexRefName": "filter-index-pattern-0", + "key": "data_stream.dataset", + "negate": false, + "params": { + "query": "aws.s3_storage_lens" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "data_stream.dataset": "aws.s3_storage_lens" + } + } + } + ], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "accessor": "893e6b97-057d-4ff1-92b0-03236debf90e", + "layerId": "af6bd20f-099d-4817-a951-99bcba7e1752", + "layerType": "data" + } + }, + "title": "S3 Storage Lens Delete Marker Object Count [AWS Metrics]", + "visualizationType": "lnsMetric" + }, + "coreMigrationVersion": "7.15.0", + "id": "aws-ed494970-41b2-11ec-a605-bff67d9b7872", + "migrationVersion": { + "lens": "7.15.0" + }, + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-af6bd20f-099d-4817-a951-99bcba7e1752", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "filter-index-pattern-0", + "type": "index-pattern" + } + ], + "type": "lens" +} \ No newline at end of file diff --git a/test/packages/aws/kibana/lens/aws-f757eba0-f0a0-11eb-b61f-a53cb4913361.json b/test/packages/aws/kibana/lens/aws-f757eba0-f0a0-11eb-b61f-a53cb4913361.json new file mode 100644 index 0000000000..fb7aff5dac --- /dev/null +++ b/test/packages/aws/kibana/lens/aws-f757eba0-f0a0-11eb-b61f-a53cb4913361.json @@ -0,0 +1,129 @@ +{ + "attributes": { + "description": null, + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "e2611df6-ca73-4d53-b0b5-afd8b718c369": { + "columnOrder": [ + "53a07fa4-b348-44c7-b644-83f3617e5b5c", + "b9e82720-e098-4dd7-ac5b-f3becccd344a", + "85528f23-48f2-462f-8075-eaddd94b21f2" + ], + "columns": { + "53a07fa4-b348-44c7-b644-83f3617e5b5c": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Read IOPS (Count/Second)", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "columnId": "85528f23-48f2-462f-8075-eaddd94b21f2", + "type": "column" + }, + "orderDirection": "desc", + "otherBucket": true, + "size": 10 + }, + "scale": "ordinal", + "sourceField": "aws.rds.db_instance.identifier" + }, + "85528f23-48f2-462f-8075-eaddd94b21f2": { + "dataType": "number", + "isBucketed": false, + "label": "Average of aws.rds.read_io.ops_per_sec", + "operationType": "average", + "scale": "ratio", + "sourceField": "aws.rds.read_io.ops_per_sec" + }, + "b9e82720-e098-4dd7-ac5b-f3becccd344a": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "layers": [ + { + "accessors": [ + "85528f23-48f2-462f-8075-eaddd94b21f2" + ], + "layerId": "e2611df6-ca73-4d53-b0b5-afd8b718c369", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "splitAccessor": "53a07fa4-b348-44c7-b644-83f3617e5b5c", + "xAccessor": "b9e82720-e098-4dd7-ac5b-f3becccd344a" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide", + "yLeftExtent": { + "mode": "full" + }, + "yRightExtent": { + "mode": "full" + } + } + }, + "title": "RDS Read IOPS [Metrics AWS]", + "visualizationType": "lnsXY" + }, + "coreMigrationVersion": "7.15.0", + "id": "aws-f757eba0-f0a0-11eb-b61f-a53cb4913361", + "migrationVersion": { + "lens": "7.14.0" + }, + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-e2611df6-ca73-4d53-b0b5-afd8b718c369", + "type": "index-pattern" + } + ], + "type": "lens" +} \ No newline at end of file diff --git a/test/packages/aws/kibana/lens/aws-f8d829f0-41a8-11ec-a605-bff67d9b7872.json b/test/packages/aws/kibana/lens/aws-f8d829f0-41a8-11ec-a605-bff67d9b7872.json new file mode 100644 index 0000000000..ff4c492c32 --- /dev/null +++ b/test/packages/aws/kibana/lens/aws-f8d829f0-41a8-11ec-a605-bff67d9b7872.json @@ -0,0 +1,216 @@ +{ + "attributes": { + "description": "", + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "eab1960c-2b9a-4e4a-9380-c29e91cbb47f": { + "columnOrder": [ + "e9704a86-fb68-4316-b885-42328390c6c0", + "a80c40e7-af1b-4ac9-ba2e-baa7c13a729a", + "f396053a-8fda-4ccb-88c5-b3ea34207c79", + "a80c40e7-af1b-4ac9-ba2e-baa7c13a729aX0", + "f396053a-8fda-4ccb-88c5-b3ea34207c79X0" + ], + "columns": { + "a80c40e7-af1b-4ac9-ba2e-baa7c13a729a": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Total Storage", + "operationType": "formula", + "params": { + "format": { + "id": "bytes", + "params": { + "decimals": 2 + } + }, + "formula": "sum(aws.s3_storage_lens.metrics.StorageBytes.avg, kql='aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" ')", + "isFormulaBroken": false + }, + "references": [ + "a80c40e7-af1b-4ac9-ba2e-baa7c13a729aX0" + ], + "scale": "ratio" + }, + "a80c40e7-af1b-4ac9-ba2e-baa7c13a729aX0": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" " + }, + "isBucketed": false, + "label": "Part of Total Storage", + "operationType": "sum", + "scale": "ratio", + "sourceField": "aws.s3_storage_lens.metrics.StorageBytes.avg" + }, + "e9704a86-fb68-4316-b885-42328390c6c0": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "AWS Regions", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "fallback": true, + "type": "alphabetical" + }, + "orderDirection": "asc", + "otherBucket": false, + "size": 5 + }, + "scale": "ordinal", + "sourceField": "aws.dimensions.aws_region" + }, + "f396053a-8fda-4ccb-88c5-b3ea34207c79": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Object Count", + "operationType": "formula", + "params": { + "formula": "sum(aws.s3_storage_lens.metrics.ObjectCount.avg, kql='aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" ')", + "isFormulaBroken": false + }, + "references": [ + "f396053a-8fda-4ccb-88c5-b3ea34207c79X0" + ], + "scale": "ratio" + }, + "f396053a-8fda-4ccb-88c5-b3ea34207c79X0": { + "customLabel": true, + "dataType": "number", + "filter": { + "language": "kuery", + "query": "aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" " + }, + "isBucketed": false, + "label": "Part of Object Count", + "operationType": "sum", + "scale": "ratio", + "sourceField": "aws.s3_storage_lens.metrics.ObjectCount.avg" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "indexRefName": "filter-index-pattern-0", + "key": "data_stream.dataset", + "negate": false, + "params": { + "query": "aws.s3_storage_lens" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "data_stream.dataset": "aws.s3_storage_lens" + } + } + } + ], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "a80c40e7-af1b-4ac9-ba2e-baa7c13a729a", + "f396053a-8fda-4ccb-88c5-b3ea34207c79" + ], + "layerId": "eab1960c-2b9a-4e4a-9380-c29e91cbb47f", + "layerType": "data", + "seriesType": "bar", + "xAccessor": "e9704a86-fb68-4316-b885-42328390c6c0", + "yConfig": [ + { + "axisMode": "left", + "color": "#e7664c", + "forAccessor": "a80c40e7-af1b-4ac9-ba2e-baa7c13a729a" + }, + { + "axisMode": "right", + "forAccessor": "f396053a-8fda-4ccb-88c5-b3ea34207c79" + } + ] + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "bar", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide", + "yLeftExtent": { + "mode": "full" + }, + "yRightExtent": { + "mode": "full" + } + } + }, + "title": "S3 Storage Lens Region Distribution [Metrics AWS]", + "visualizationType": "lnsXY" + }, + "coreMigrationVersion": "7.15.0", + "id": "aws-f8d829f0-41a8-11ec-a605-bff67d9b7872", + "migrationVersion": { + "lens": "7.15.0" + }, + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-eab1960c-2b9a-4e4a-9380-c29e91cbb47f", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "filter-index-pattern-0", + "type": "index-pattern" + } + ], + "type": "lens" +} \ No newline at end of file diff --git a/test/packages/aws/kibana/lens/aws-f9ff3450-f094-11eb-b61f-a53cb4913361.json b/test/packages/aws/kibana/lens/aws-f9ff3450-f094-11eb-b61f-a53cb4913361.json new file mode 100644 index 0000000000..d91315518c --- /dev/null +++ b/test/packages/aws/kibana/lens/aws-f9ff3450-f094-11eb-b61f-a53cb4913361.json @@ -0,0 +1,145 @@ +{ + "attributes": { + "description": null, + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "5a1e8135-28e5-4e15-a675-bf9f840fca1c": { + "columnOrder": [ + "748501ab-c222-4695-9062-93c70e72a864", + "191b027e-d3d1-41da-b3fb-29315f99f4ee", + "98446733-f0c0-4666-8491-52b6eae923ed", + "98446733-f0c0-4666-8491-52b6eae923edX0" + ], + "columns": { + "191b027e-d3d1-41da-b3fb-29315f99f4ee": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp" + }, + "748501ab-c222-4695-9062-93c70e72a864": { + "dataType": "string", + "isBucketed": true, + "label": "Top values of aws.rds.db_instance.identifier", + "operationType": "terms", + "params": { + "missingBucket": false, + "orderBy": { + "fallback": true, + "type": "alphabetical" + }, + "orderDirection": "asc", + "otherBucket": true, + "size": 10 + }, + "scale": "ordinal", + "sourceField": "aws.rds.db_instance.identifier" + }, + "98446733-f0c0-4666-8491-52b6eae923ed": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Read Latency (Milliseconds)", + "operationType": "formula", + "params": { + "formula": "average(aws.rds.latency.read)", + "isFormulaBroken": false + }, + "references": [ + "98446733-f0c0-4666-8491-52b6eae923edX0" + ], + "scale": "ratio" + }, + "98446733-f0c0-4666-8491-52b6eae923edX0": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Part of average(aws.rds.latency.read)", + "operationType": "average", + "scale": "ratio", + "sourceField": "aws.rds.latency.read" + } + }, + "incompleteColumns": {} + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "layers": [ + { + "accessors": [ + "98446733-f0c0-4666-8491-52b6eae923ed" + ], + "layerId": "5a1e8135-28e5-4e15-a675-bf9f840fca1c", + "position": "top", + "seriesType": "line", + "showGridlines": false, + "splitAccessor": "748501ab-c222-4695-9062-93c70e72a864", + "xAccessor": "191b027e-d3d1-41da-b3fb-29315f99f4ee" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "line", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide", + "yLeftExtent": { + "mode": "full" + }, + "yRightExtent": { + "mode": "full" + } + } + }, + "title": "RDS Read Latency in Milliseconds [Metrics AWS]", + "visualizationType": "lnsXY" + }, + "coreMigrationVersion": "7.15.0", + "id": "aws-f9ff3450-f094-11eb-b61f-a53cb4913361", + "migrationVersion": { + "lens": "7.14.0" + }, + "references": [ + { + "id": "metrics-*", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "indexpattern-datasource-layer-5a1e8135-28e5-4e15-a675-bf9f840fca1c", + "type": "index-pattern" + } + ], + "type": "lens" +} \ No newline at end of file diff --git a/test/packages/aws/kibana/visualization/aws-00b29040-921d-11e9-aa19-159bf182e06f.json b/test/packages/aws/kibana/visualization/aws-00b29040-921d-11e9-aa19-159bf182e06f.json index f9da2243bd..c04ab0482a 100644 --- a/test/packages/aws/kibana/visualization/aws-00b29040-921d-11e9-aa19-159bf182e06f.json +++ b/test/packages/aws/kibana/visualization/aws-00b29040-921d-11e9-aa19-159bf182e06f.json @@ -2,13 +2,7 @@ "attributes": { "description": "", "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [], - "query": { - "language": "kuery", - "query": "" - } - } + "searchSourceJSON": {} }, "title": "RDS Transaction Blocked [Metrics AWS]", "uiStateJSON": {}, @@ -31,6 +25,18 @@ "id": "3526a9e0-d978-11e9-aff2-99c15d8b7da1", "operator": "gt", "value": 0 + }, + { + "background_color": "rgba(164,221,0,1)", + "id": "50a46c15-ab5f-41d6-83de-b988ab7f4149", + "operator": "empty", + "value": null + }, + { + "background_color": "rgba(164,221,0,1)", + "id": "a42d8157-775b-4e45-bf79-b3b572e2235b", + "operator": "empty", + "value": null } ], "bar_color_rules": [ @@ -45,6 +51,7 @@ "default_timefield": "@timestamp", "drilldown_url": "", "filter": "", + "hide_last_value_indicator": true, "id": "61ca57f0-469d-11e7-af02-69e470af7417", "index_pattern": "metrics-*", "interval": "1m", @@ -79,13 +86,18 @@ "show_grid": 1, "show_legend": 1, "time_field": "", - "type": "metric" + "type": "metric", + "use_kibana_indexes": false }, "title": "RDS Transaction Blocked [Metrics AWS]", "type": "metrics" } }, + "coreMigrationVersion": "7.15.0", "id": "aws-00b29040-921d-11e9-aa19-159bf182e06f", + "migrationVersion": { + "visualization": "7.14.0" + }, "references": [], "type": "visualization" } \ No newline at end of file diff --git a/test/packages/aws/kibana/visualization/aws-11f4e0d0-42a9-11ec-a440-675e476c18bb.json b/test/packages/aws/kibana/visualization/aws-11f4e0d0-42a9-11ec-a440-675e476c18bb.json new file mode 100644 index 0000000000..9ca9794566 --- /dev/null +++ b/test/packages/aws/kibana/visualization/aws-11f4e0d0-42a9-11ec-a440-675e476c18bb.json @@ -0,0 +1,107 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "S3 Storage Lens Total Storage Region Distribution [AWS Metrics]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "background_color_rules": [ + { + "id": "88f52970-42a8-11ec-afe1-bb42ec488e9b" + } + ], + "bar_color_rules": [ + { + "id": "5fac2960-42a8-11ec-afe1-bb42ec488e9b" + } + ], + "drop_last_bucket": 0, + "filter": { + "language": "kuery", + "query": "aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" " + }, + "gauge_color_rules": [ + { + "id": "864db020-42a8-11ec-afe1-bb42ec488e9b" + } + ], + "gauge_inner_width": 10, + "gauge_style": "half", + "gauge_width": 10, + "id": "c08f117f-f0e8-4d10-ace7-5d2a5aab35c0", + "index_pattern_ref_name": "metrics_0_index_pattern", + "interval": "3d", + "isModelInvalid": false, + "max_lines_legend": 1, + "series": [ + { + "axis_position": "left", + "chart_type": "bar", + "color": "rgba(96,146,192,1)", + "fill": "1", + "formatter": "bytes", + "id": "de4a05bf-2a2d-422d-98ba-49350f7442d9", + "label": "Total Storage", + "line_width": 1, + "metrics": [ + { + "field": "aws.s3_storage_lens.metrics.StorageBytes.avg", + "id": "cf9f8d20-f581-4036-8f5f-07748b6f9533", + "type": "sum" + } + ], + "palette": { + "name": "default", + "type": "palette" + }, + "point_size": 1, + "separate_axis": 1, + "split_mode": "terms", + "stacked": "none", + "terms_field": "aws.dimensions.aws_region", + "terms_order_by": "cf9f8d20-f581-4036-8f5f-07748b6f9533", + "terms_size": "5", + "type": "timeseries" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "", + "time_range_mode": "last_value", + "tooltip_mode": "show_all", + "truncate_legend": 1, + "type": "top_n", + "use_kibana_indexes": true + }, + "title": "S3 Storage Lens Total Storage Region Distribution [AWS Metrics]", + "type": "metrics" + } + }, + "coreMigrationVersion": "7.15.0", + "id": "aws-11f4e0d0-42a9-11ec-a440-675e476c18bb", + "migrationVersion": { + "visualization": "7.14.0" + }, + "references": [ + { + "id": "metrics-*", + "name": "metrics_0_index_pattern", + "type": "index-pattern" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/aws/kibana/visualization/aws-17fcda50-921b-11e9-aa19-159bf182e06f.json b/test/packages/aws/kibana/visualization/aws-17fcda50-921b-11e9-aa19-159bf182e06f.json index f6e0b2500b..0ef28431c6 100644 --- a/test/packages/aws/kibana/visualization/aws-17fcda50-921b-11e9-aa19-159bf182e06f.json +++ b/test/packages/aws/kibana/visualization/aws-17fcda50-921b-11e9-aa19-159bf182e06f.json @@ -2,13 +2,7 @@ "attributes": { "description": "", "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [], - "query": { - "language": "kuery", - "query": "" - } - } + "searchSourceJSON": {} }, "title": "RDS Database Connections [Metrics AWS]", "uiStateJSON": {}, @@ -65,13 +59,18 @@ "show_grid": 1, "show_legend": 1, "time_field": "", - "type": "metric" + "type": "metric", + "use_kibana_indexes": false }, "title": "RDS Database Connections [Metrics AWS]", "type": "metrics" } }, + "coreMigrationVersion": "7.15.0", "id": "aws-17fcda50-921b-11e9-aa19-159bf182e06f", + "migrationVersion": { + "visualization": "7.14.0" + }, "references": [], "type": "visualization" } \ No newline at end of file diff --git a/test/packages/aws/kibana/visualization/aws-2e265070-42a6-11ec-a440-675e476c18bb.json b/test/packages/aws/kibana/visualization/aws-2e265070-42a6-11ec-a440-675e476c18bb.json new file mode 100644 index 0000000000..916fcaa7fd --- /dev/null +++ b/test/packages/aws/kibana/visualization/aws-2e265070-42a6-11ec-a440-675e476c18bb.json @@ -0,0 +1,121 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "S3 Storage Lens Average Object Size[Metrics AWS]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "background_color_rules": [ + { + "id": "65c0c4b0-42a2-11ec-afe1-bb42ec488e9b" + } + ], + "drop_last_bucket": 0, + "id": "142ea8b7-6859-4f47-a442-71e4c5995e8d", + "index_pattern_ref_name": "metrics_0_index_pattern", + "interval": "", + "isModelInvalid": false, + "max_lines_legend": 1, + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "filter": { + "language": "kuery", + "query": "aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" " + }, + "formatter": "bytes", + "id": "de4a05bf-2a2d-422d-98ba-49350f7442d9", + "label": "Average Object Size", + "line_width": 1, + "metrics": [ + { + "field": "aws.s3_storage_lens.metrics.ObjectCount.avg", + "id": "4074b470-42a5-11ec-9ec9-3315b0d7914c", + "type": "sum" + }, + { + "field": "aws.s3_storage_lens.metrics.StorageBytes.avg", + "id": "cf9f8d20-f581-4036-8f5f-07748b6f9533", + "type": "sum" + }, + { + "id": "568cd530-42a5-11ec-9ec9-3315b0d7914c", + "script": "divide(params.TotalStorage, params.ObjectCount)", + "type": "math", + "variables": [ + { + "field": "cf9f8d20-f581-4036-8f5f-07748b6f9533", + "id": "59e1cc90-42a5-11ec-9ec9-3315b0d7914c", + "name": "TotalStorage" + }, + { + "field": "4074b470-42a5-11ec-9ec9-3315b0d7914c", + "id": "64d21790-42a5-11ec-9ec9-3315b0d7914c", + "name": "ObjectCount" + } + ] + } + ], + "override_index_pattern": 1, + "palette": { + "name": "default", + "type": "palette" + }, + "point_size": 1, + "separate_axis": 0, + "series_index_pattern_ref_name": "metrics_1_index_pattern", + "series_interval": "3d", + "split_mode": "everything", + "stacked": "none", + "time_range_mode": "last_value" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "", + "time_range_mode": "entire_time_range", + "tooltip_mode": "show_all", + "truncate_legend": 1, + "type": "metric", + "use_kibana_indexes": true + }, + "title": "S3 Storage Lens Average Object Size[Metrics AWS]", + "type": "metrics" + } + }, + "coreMigrationVersion": "7.15.0", + "id": "aws-2e265070-42a6-11ec-a440-675e476c18bb", + "migrationVersion": { + "visualization": "7.14.0" + }, + "references": [ + { + "id": "metrics-*", + "name": "metrics_0_index_pattern", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "metrics_1_index_pattern", + "type": "index-pattern" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/aws/kibana/visualization/aws-37aeedc0-42a9-11ec-a440-675e476c18bb.json b/test/packages/aws/kibana/visualization/aws-37aeedc0-42a9-11ec-a440-675e476c18bb.json new file mode 100644 index 0000000000..a1345159a0 --- /dev/null +++ b/test/packages/aws/kibana/visualization/aws-37aeedc0-42a9-11ec-a440-675e476c18bb.json @@ -0,0 +1,107 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "S3 Storage Lens Object Count Region Distribution [AWS Metrics]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "background_color_rules": [ + { + "id": "88f52970-42a8-11ec-afe1-bb42ec488e9b" + } + ], + "bar_color_rules": [ + { + "id": "5fac2960-42a8-11ec-afe1-bb42ec488e9b" + } + ], + "drop_last_bucket": 0, + "filter": { + "language": "kuery", + "query": "aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" " + }, + "gauge_color_rules": [ + { + "id": "864db020-42a8-11ec-afe1-bb42ec488e9b" + } + ], + "gauge_inner_width": 10, + "gauge_style": "half", + "gauge_width": 10, + "id": "c08f117f-f0e8-4d10-ace7-5d2a5aab35c0", + "index_pattern_ref_name": "metrics_0_index_pattern", + "interval": "3d", + "isModelInvalid": false, + "max_lines_legend": 1, + "series": [ + { + "axis_position": "left", + "chart_type": "bar", + "color": "rgba(231,102,76,1)", + "fill": "1", + "formatter": "number", + "id": "de4a05bf-2a2d-422d-98ba-49350f7442d9", + "label": "Object Count", + "line_width": 1, + "metrics": [ + { + "field": "aws.s3_storage_lens.metrics.ObjectCount.avg", + "id": "cf9f8d20-f581-4036-8f5f-07748b6f9533", + "type": "sum" + } + ], + "palette": { + "name": "default", + "type": "palette" + }, + "point_size": 1, + "separate_axis": 1, + "split_mode": "terms", + "stacked": "none", + "terms_field": "aws.dimensions.aws_region", + "terms_order_by": "cf9f8d20-f581-4036-8f5f-07748b6f9533", + "terms_size": "5", + "type": "timeseries" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "", + "time_range_mode": "last_value", + "tooltip_mode": "show_all", + "truncate_legend": 1, + "type": "top_n", + "use_kibana_indexes": true + }, + "title": "S3 Storage Lens Object Count Region Distribution [AWS Metrics]", + "type": "metrics" + } + }, + "coreMigrationVersion": "7.15.0", + "id": "aws-37aeedc0-42a9-11ec-a440-675e476c18bb", + "migrationVersion": { + "visualization": "7.14.0" + }, + "references": [ + { + "id": "metrics-*", + "name": "metrics_0_index_pattern", + "type": "index-pattern" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/aws/kibana/visualization/aws-3c08b9b0-42a3-11ec-a440-675e476c18bb.json b/test/packages/aws/kibana/visualization/aws-3c08b9b0-42a3-11ec-a440-675e476c18bb.json new file mode 100644 index 0000000000..073e449a73 --- /dev/null +++ b/test/packages/aws/kibana/visualization/aws-3c08b9b0-42a3-11ec-a440-675e476c18bb.json @@ -0,0 +1,99 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "S3 Storage Lens Total Storage [Metrics AWS]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "background_color_rules": [ + { + "id": "65c0c4b0-42a2-11ec-afe1-bb42ec488e9b" + } + ], + "drop_last_bucket": 0, + "id": "142ea8b7-6859-4f47-a442-71e4c5995e8d", + "index_pattern_ref_name": "metrics_0_index_pattern", + "interval": "", + "isModelInvalid": false, + "max_lines_legend": 1, + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "filter": { + "language": "kuery", + "query": "aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" " + }, + "formatter": "bytes", + "id": "de4a05bf-2a2d-422d-98ba-49350f7442d9", + "label": "Total Storage", + "line_width": 1, + "metrics": [ + { + "field": "aws.s3_storage_lens.metrics.StorageBytes.avg", + "id": "cf9f8d20-f581-4036-8f5f-07748b6f9533", + "type": "sum" + } + ], + "override_index_pattern": 1, + "palette": { + "name": "default", + "type": "palette" + }, + "point_size": 1, + "separate_axis": 0, + "series_index_pattern_ref_name": "metrics_1_index_pattern", + "series_interval": "3d", + "split_mode": "everything", + "stacked": "none", + "time_range_mode": "last_value" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "", + "time_range_mode": "entire_time_range", + "tooltip_mode": "show_all", + "truncate_legend": 1, + "type": "metric", + "use_kibana_indexes": true + }, + "title": "S3 Storage Lens Total Storage [Metrics AWS]", + "type": "metrics" + } + }, + "coreMigrationVersion": "7.15.0", + "id": "aws-3c08b9b0-42a3-11ec-a440-675e476c18bb", + "migrationVersion": { + "visualization": "7.14.0" + }, + "references": [ + { + "id": "metrics-*", + "name": "metrics_0_index_pattern", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "metrics_1_index_pattern", + "type": "index-pattern" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/aws/kibana/visualization/aws-41f5aaf0-42ac-11ec-a440-675e476c18bb.json b/test/packages/aws/kibana/visualization/aws-41f5aaf0-42ac-11ec-a440-675e476c18bb.json new file mode 100644 index 0000000000..daede05d35 --- /dev/null +++ b/test/packages/aws/kibana/visualization/aws-41f5aaf0-42ac-11ec-a440-675e476c18bb.json @@ -0,0 +1,99 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "S3 Storage Lens Delete Marker Object Count [AWS Metrics]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "background_color_rules": [ + { + "id": "66732f50-42a3-11ec-afe1-bb42ec488e9b" + } + ], + "drop_last_bucket": 0, + "id": "c9efe17d-a8ee-4317-8eaa-e00070f8c4f4", + "index_pattern_ref_name": "metrics_0_index_pattern", + "interval": "", + "isModelInvalid": false, + "max_lines_legend": 1, + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "filter": { + "language": "kuery", + "query": "aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" " + }, + "formatter": "number", + "id": "de4a05bf-2a2d-422d-98ba-49350f7442d9", + "label": "Delete Marker Object Count", + "line_width": 1, + "metrics": [ + { + "field": "aws.s3_storage_lens.metrics.DeleteMarkerObjectCount.avg", + "id": "cf9f8d20-f581-4036-8f5f-07748b6f9533", + "type": "sum" + } + ], + "override_index_pattern": 1, + "palette": { + "name": "default", + "type": "palette" + }, + "point_size": 1, + "separate_axis": 0, + "series_index_pattern_ref_name": "metrics_1_index_pattern", + "series_interval": "3d", + "split_mode": "everything", + "stacked": "none", + "type": "timeseries" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "", + "time_range_mode": "entire_time_range", + "tooltip_mode": "show_all", + "truncate_legend": 1, + "type": "metric", + "use_kibana_indexes": true + }, + "title": "S3 Storage Lens Delete Marker Object Count [AWS Metrics]", + "type": "metrics" + } + }, + "coreMigrationVersion": "7.15.0", + "id": "aws-41f5aaf0-42ac-11ec-a440-675e476c18bb", + "migrationVersion": { + "visualization": "7.14.0" + }, + "references": [ + { + "id": "metrics-*", + "name": "metrics_0_index_pattern", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "metrics_1_index_pattern", + "type": "index-pattern" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/aws/kibana/visualization/aws-526a1ba0-42a7-11ec-a440-675e476c18bb.json b/test/packages/aws/kibana/visualization/aws-526a1ba0-42a7-11ec-a440-675e476c18bb.json new file mode 100644 index 0000000000..22d68b686a --- /dev/null +++ b/test/packages/aws/kibana/visualization/aws-526a1ba0-42a7-11ec-a440-675e476c18bb.json @@ -0,0 +1,99 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "S3 Storage Lens Active Buckets [AWS Metrics]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "background_color_rules": [ + { + "id": "6479f000-42a6-11ec-afe1-bb42ec488e9b" + } + ], + "drop_last_bucket": 0, + "id": "2e804eee-436c-4e36-917b-7400ea5e5f89", + "index_pattern_ref_name": "metrics_0_index_pattern", + "interval": "", + "isModelInvalid": false, + "max_lines_legend": 1, + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "filter": { + "language": "kuery", + "query": "not aws.dimensions.storage_class : \"-\" " + }, + "formatter": "number", + "id": "de4a05bf-2a2d-422d-98ba-49350f7442d9", + "label": "Active Buckets", + "line_width": 1, + "metrics": [ + { + "field": "aws.dimensions.bucket_name", + "id": "cf9f8d20-f581-4036-8f5f-07748b6f9533", + "type": "cardinality" + } + ], + "override_index_pattern": 1, + "palette": { + "name": "default", + "type": "palette" + }, + "point_size": 1, + "separate_axis": 0, + "series_index_pattern_ref_name": "metrics_1_index_pattern", + "series_interval": "3d", + "split_mode": "everything", + "stacked": "none", + "type": "timeseries" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "", + "time_range_mode": "entire_time_range", + "tooltip_mode": "show_all", + "truncate_legend": 1, + "type": "metric", + "use_kibana_indexes": true + }, + "title": "S3 Storage Lens Active Buckets [AWS Metrics]", + "type": "metrics" + } + }, + "coreMigrationVersion": "7.15.0", + "id": "aws-526a1ba0-42a7-11ec-a440-675e476c18bb", + "migrationVersion": { + "visualization": "7.14.0" + }, + "references": [ + { + "id": "metrics-*", + "name": "metrics_0_index_pattern", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "metrics_1_index_pattern", + "type": "index-pattern" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/aws/kibana/visualization/aws-57b610a0-41ad-11ec-a605-bff67d9b7872.json b/test/packages/aws/kibana/visualization/aws-57b610a0-41ad-11ec-a605-bff67d9b7872.json new file mode 100644 index 0000000000..7a8a426400 --- /dev/null +++ b/test/packages/aws/kibana/visualization/aws-57b610a0-41ad-11ec-a605-bff67d9b7872.json @@ -0,0 +1,97 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "S3 Storage Lens Filters [Metrics AWS]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "controls": [ + { + "fieldName": "aws.dimensions.aws_account_number", + "id": "1636497293904", + "indexPatternRefName": "control_0_index_pattern", + "label": "Account", + "options": { + "dynamicOptions": true, + "multiselect": true, + "order": "desc", + "size": 5, + "type": "terms" + }, + "parent": "", + "type": "list" + }, + { + "fieldName": "aws.dimensions.aws_region", + "id": "1549397251041", + "indexPatternRefName": "control_1_index_pattern", + "label": "Region", + "options": { + "dynamicOptions": true, + "multiselect": true, + "order": "desc", + "size": 5, + "type": "terms" + }, + "parent": "", + "type": "list" + }, + { + "fieldName": "aws.dimensions.bucket_name", + "id": "1549512142947", + "indexPatternRefName": "control_2_index_pattern", + "label": "S3 Bucket Name", + "options": { + "dynamicOptions": true, + "multiselect": true, + "order": "desc", + "size": 5, + "type": "terms" + }, + "parent": "", + "type": "list" + } + ], + "pinFilters": false, + "updateFiltersOnChange": true, + "useTimeFilter": false + }, + "title": "S3 Storage Lens Filters [Metrics AWS]", + "type": "input_control_vis" + } + }, + "coreMigrationVersion": "7.15.0", + "id": "aws-57b610a0-41ad-11ec-a605-bff67d9b7872", + "migrationVersion": { + "visualization": "7.14.0" + }, + "references": [ + { + "id": "metrics-*", + "name": "control_0_index_pattern", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "control_1_index_pattern", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "control_2_index_pattern", + "type": "index-pattern" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/aws/kibana/visualization/aws-5c586e80-427c-11ec-9b39-b9df88395505.json b/test/packages/aws/kibana/visualization/aws-5c586e80-427c-11ec-9b39-b9df88395505.json new file mode 100644 index 0000000000..18a8faa53e --- /dev/null +++ b/test/packages/aws/kibana/visualization/aws-5c586e80-427c-11ec-9b39-b9df88395505.json @@ -0,0 +1,34 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "S3 Storage Lens Cost Efficiency Markdown [AWS Metrics]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "fontSize": 14, + "markdown": "Cost Efficiency", + "openLinksInNewTab": false + }, + "title": "S3 Storage Lens Cost Efficiency Markdown [AWS Metrics]", + "type": "markdown" + } + }, + "coreMigrationVersion": "7.15.0", + "id": "aws-5c586e80-427c-11ec-9b39-b9df88395505", + "migrationVersion": { + "visualization": "7.14.0" + }, + "references": [], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/aws/kibana/visualization/aws-7593a130-427c-11ec-9b39-b9df88395505.json b/test/packages/aws/kibana/visualization/aws-7593a130-427c-11ec-9b39-b9df88395505.json new file mode 100644 index 0000000000..8fce91d111 --- /dev/null +++ b/test/packages/aws/kibana/visualization/aws-7593a130-427c-11ec-9b39-b9df88395505.json @@ -0,0 +1,34 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "S3 Storage Lens Trends Markdown [AWS Metrics]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "fontSize": 16, + "markdown": "Trends", + "openLinksInNewTab": false + }, + "title": "S3 Storage Lens Trends Markdown [AWS Metrics]", + "type": "markdown" + } + }, + "coreMigrationVersion": "7.15.0", + "id": "aws-7593a130-427c-11ec-9b39-b9df88395505", + "migrationVersion": { + "visualization": "7.14.0" + }, + "references": [], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/aws/kibana/visualization/aws-7dd7fab0-42ab-11ec-a440-675e476c18bb.json b/test/packages/aws/kibana/visualization/aws-7dd7fab0-42ab-11ec-a440-675e476c18bb.json new file mode 100644 index 0000000000..1bad4aecbf --- /dev/null +++ b/test/packages/aws/kibana/visualization/aws-7dd7fab0-42ab-11ec-a440-675e476c18bb.json @@ -0,0 +1,127 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "S3 Storage Lens Replicated Bytes Percentage [AWS Metrics]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "background_color_rules": [ + { + "id": "8d0ce9b0-42aa-11ec-93b6-03cae606ce61" + } + ], + "bar_color_rules": [ + { + "id": "ddaf4ed0-42aa-11ec-a5a1-73d3b46a933e" + } + ], + "drop_last_bucket": 0, + "filter": { + "language": "kuery", + "query": "aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" " + }, + "gauge_color_rules": [ + { + "id": "0d38a660-42ab-11ec-a5a1-73d3b46a933e" + } + ], + "gauge_inner_width": 10, + "gauge_style": "half", + "gauge_width": 10, + "id": "7c0ba441-be60-42ec-9d68-6959f1c1e709", + "index_pattern_ref_name": "metrics_0_index_pattern", + "interval": "", + "isModelInvalid": false, + "markdown": "test\t{{ object_lock_bytes_percentage.last.raw }}", + "max_lines_legend": 1, + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "formatter": "percent", + "id": "4e7561ec-a29c-4558-9d40-d8f378c85daf", + "label": "Replicated", + "line_width": 1, + "metrics": [ + { + "field": "aws.s3_storage_lens.metrics.ReplicatedStorageBytes.avg", + "id": "5e067d85-0dfc-4208-a72b-75279b58b2b2", + "type": "sum" + }, + { + "field": "aws.s3_storage_lens.metrics.StorageBytes.avg", + "id": "66645e10-42aa-11ec-93b6-03cae606ce61", + "type": "sum" + }, + { + "id": "707b8720-42aa-11ec-93b6-03cae606ce61", + "script": "divide(params.Replicated, params.StorageBytes)", + "type": "math", + "variables": [ + { + "field": "5e067d85-0dfc-4208-a72b-75279b58b2b2", + "id": "73b2bd50-42aa-11ec-93b6-03cae606ce61", + "name": "Replicated" + }, + { + "field": "66645e10-42aa-11ec-93b6-03cae606ce61", + "id": "7c32cba0-42aa-11ec-93b6-03cae606ce61", + "name": "StorageBytes" + } + ] + } + ], + "palette": { + "name": "default", + "type": "palette" + }, + "point_size": 1, + "separate_axis": 0, + "split_mode": "everything", + "stacked": "none", + "type": "timeseries" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "", + "time_range_mode": "entire_time_range", + "tooltip_mode": "show_all", + "truncate_legend": 1, + "type": "metric", + "use_kibana_indexes": true + }, + "title": "S3 Storage Lens Replicated Bytes Percentage [AWS Metrics]", + "type": "metrics" + } + }, + "coreMigrationVersion": "7.15.0", + "id": "aws-7dd7fab0-42ab-11ec-a440-675e476c18bb", + "migrationVersion": { + "visualization": "7.14.0" + }, + "references": [ + { + "id": "metrics-*", + "name": "metrics_0_index_pattern", + "type": "index-pattern" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/aws/kibana/visualization/aws-82b8f4a0-427c-11ec-9b39-b9df88395505.json b/test/packages/aws/kibana/visualization/aws-82b8f4a0-427c-11ec-9b39-b9df88395505.json new file mode 100644 index 0000000000..20efe326d5 --- /dev/null +++ b/test/packages/aws/kibana/visualization/aws-82b8f4a0-427c-11ec-9b39-b9df88395505.json @@ -0,0 +1,34 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "S3 Storage Lens Daily Overview Markdown [AWS Metrics]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "fontSize": 16, + "markdown": "Daily Overview", + "openLinksInNewTab": false + }, + "title": "S3 Storage Lens Daily Overview Markdown [AWS Metrics]", + "type": "markdown" + } + }, + "coreMigrationVersion": "7.15.0", + "id": "aws-82b8f4a0-427c-11ec-9b39-b9df88395505", + "migrationVersion": { + "visualization": "7.14.0" + }, + "references": [], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/aws/kibana/visualization/aws-96834640-42a7-11ec-a440-675e476c18bb.json b/test/packages/aws/kibana/visualization/aws-96834640-42a7-11ec-a440-675e476c18bb.json new file mode 100644 index 0000000000..3f710ff414 --- /dev/null +++ b/test/packages/aws/kibana/visualization/aws-96834640-42a7-11ec-a440-675e476c18bb.json @@ -0,0 +1,99 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "S3 Storage Lens Accounts [AWS Metrics]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "background_color_rules": [ + { + "id": "6479f000-42a6-11ec-afe1-bb42ec488e9b" + } + ], + "drop_last_bucket": 0, + "id": "2e804eee-436c-4e36-917b-7400ea5e5f89", + "index_pattern_ref_name": "metrics_0_index_pattern", + "interval": "", + "isModelInvalid": false, + "max_lines_legend": 1, + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "filter": { + "language": "kuery", + "query": "not aws.dimensions.storage_class : \"-\" " + }, + "formatter": "number", + "id": "de4a05bf-2a2d-422d-98ba-49350f7442d9", + "label": "Accounts", + "line_width": 1, + "metrics": [ + { + "field": "aws.dimensions.aws_account_number", + "id": "cf9f8d20-f581-4036-8f5f-07748b6f9533", + "type": "cardinality" + } + ], + "override_index_pattern": 1, + "palette": { + "name": "default", + "type": "palette" + }, + "point_size": 1, + "separate_axis": 0, + "series_index_pattern_ref_name": "metrics_1_index_pattern", + "series_interval": "3d", + "split_mode": "everything", + "stacked": "none", + "type": "timeseries" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "", + "time_range_mode": "entire_time_range", + "tooltip_mode": "show_all", + "truncate_legend": 1, + "type": "metric", + "use_kibana_indexes": true + }, + "title": "S3 Storage Lens Accounts [AWS Metrics]", + "type": "metrics" + } + }, + "coreMigrationVersion": "7.15.0", + "id": "aws-96834640-42a7-11ec-a440-675e476c18bb", + "migrationVersion": { + "visualization": "7.14.0" + }, + "references": [ + { + "id": "metrics-*", + "name": "metrics_0_index_pattern", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "metrics_1_index_pattern", + "type": "index-pattern" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/aws/kibana/visualization/aws-a1670af0-42a3-11ec-a440-675e476c18bb.json b/test/packages/aws/kibana/visualization/aws-a1670af0-42a3-11ec-a440-675e476c18bb.json new file mode 100644 index 0000000000..3963300e90 --- /dev/null +++ b/test/packages/aws/kibana/visualization/aws-a1670af0-42a3-11ec-a440-675e476c18bb.json @@ -0,0 +1,99 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "S3 Storage Lens Object Count [AWS Metrics]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "background_color_rules": [ + { + "id": "66732f50-42a3-11ec-afe1-bb42ec488e9b" + } + ], + "drop_last_bucket": 0, + "id": "c9efe17d-a8ee-4317-8eaa-e00070f8c4f4", + "index_pattern_ref_name": "metrics_0_index_pattern", + "interval": "", + "isModelInvalid": false, + "max_lines_legend": 1, + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "filter": { + "language": "kuery", + "query": "aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" " + }, + "formatter": "number", + "id": "de4a05bf-2a2d-422d-98ba-49350f7442d9", + "label": "Object Count", + "line_width": 1, + "metrics": [ + { + "field": "aws.s3_storage_lens.metrics.ObjectCount.avg", + "id": "cf9f8d20-f581-4036-8f5f-07748b6f9533", + "type": "sum" + } + ], + "override_index_pattern": 1, + "palette": { + "name": "default", + "type": "palette" + }, + "point_size": 1, + "separate_axis": 0, + "series_index_pattern_ref_name": "metrics_1_index_pattern", + "series_interval": "3d", + "split_mode": "everything", + "stacked": "none", + "type": "timeseries" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "", + "time_range_mode": "entire_time_range", + "tooltip_mode": "show_all", + "truncate_legend": 1, + "type": "metric", + "use_kibana_indexes": true + }, + "title": "S3 Storage Lens Object Count [AWS Metrics]", + "type": "metrics" + } + }, + "coreMigrationVersion": "7.15.0", + "id": "aws-a1670af0-42a3-11ec-a440-675e476c18bb", + "migrationVersion": { + "visualization": "7.14.0" + }, + "references": [ + { + "id": "metrics-*", + "name": "metrics_0_index_pattern", + "type": "index-pattern" + }, + { + "id": "metrics-*", + "name": "metrics_1_index_pattern", + "type": "index-pattern" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/aws/kibana/visualization/aws-b5308940-7347-11e9-816b-07687310a99a.json b/test/packages/aws/kibana/visualization/aws-b5308940-7347-11e9-816b-07687310a99a.json index fa5f69ce36..ed8d9fac3f 100644 --- a/test/packages/aws/kibana/visualization/aws-b5308940-7347-11e9-816b-07687310a99a.json +++ b/test/packages/aws/kibana/visualization/aws-b5308940-7347-11e9-816b-07687310a99a.json @@ -10,7 +10,7 @@ } } }, - "title": "AWS Region Filter", + "title": "AWS Region Filter [Metrics AWS]", "uiStateJSON": {}, "version": 1, "visState": { @@ -41,7 +41,11 @@ "type": "input_control_vis" } }, + "coreMigrationVersion": "7.15.0", "id": "aws-b5308940-7347-11e9-816b-07687310a99a", + "migrationVersion": { + "visualization": "7.14.0" + }, "references": [ { "id": "metrics-*", diff --git a/test/packages/aws/kibana/visualization/aws-c4a82470-42aa-11ec-a440-675e476c18bb.json b/test/packages/aws/kibana/visualization/aws-c4a82470-42aa-11ec-a440-675e476c18bb.json new file mode 100644 index 0000000000..a4edc70fdf --- /dev/null +++ b/test/packages/aws/kibana/visualization/aws-c4a82470-42aa-11ec-a440-675e476c18bb.json @@ -0,0 +1,127 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "S3 Storage Lens Object Lock Bytes Percentage [AWS Metrics]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "background_color_rules": [ + { + "id": "8d0ce9b0-42aa-11ec-93b6-03cae606ce61" + } + ], + "bar_color_rules": [ + { + "id": "ddaf4ed0-42aa-11ec-a5a1-73d3b46a933e" + } + ], + "drop_last_bucket": 0, + "filter": { + "language": "kuery", + "query": "aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" " + }, + "gauge_color_rules": [ + { + "id": "0d38a660-42ab-11ec-a5a1-73d3b46a933e" + } + ], + "gauge_inner_width": 10, + "gauge_style": "half", + "gauge_width": 10, + "id": "7c0ba441-be60-42ec-9d68-6959f1c1e709", + "index_pattern_ref_name": "metrics_0_index_pattern", + "interval": "", + "isModelInvalid": false, + "markdown": "test\t{{ object_lock_bytes_percentage.last.raw }}", + "max_lines_legend": 1, + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#68BC00", + "fill": 0.5, + "formatter": "percent", + "id": "4e7561ec-a29c-4558-9d40-d8f378c85daf", + "label": "Object Locked", + "line_width": 1, + "metrics": [ + { + "field": "aws.s3_storage_lens.metrics.ObjectLockEnabledStorageBytes.avg", + "id": "5e067d85-0dfc-4208-a72b-75279b58b2b2", + "type": "sum" + }, + { + "field": "aws.s3_storage_lens.metrics.StorageBytes.avg", + "id": "66645e10-42aa-11ec-93b6-03cae606ce61", + "type": "sum" + }, + { + "id": "707b8720-42aa-11ec-93b6-03cae606ce61", + "script": "divide(params.ObjectLock, params.StorageBytes)", + "type": "math", + "variables": [ + { + "field": "5e067d85-0dfc-4208-a72b-75279b58b2b2", + "id": "73b2bd50-42aa-11ec-93b6-03cae606ce61", + "name": "ObjectLock" + }, + { + "field": "66645e10-42aa-11ec-93b6-03cae606ce61", + "id": "7c32cba0-42aa-11ec-93b6-03cae606ce61", + "name": "StorageBytes" + } + ] + } + ], + "palette": { + "name": "default", + "type": "palette" + }, + "point_size": 1, + "separate_axis": 0, + "split_mode": "everything", + "stacked": "none", + "type": "timeseries" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "", + "time_range_mode": "entire_time_range", + "tooltip_mode": "show_all", + "truncate_legend": 1, + "type": "metric", + "use_kibana_indexes": true + }, + "title": "S3 Storage Lens Object Lock Bytes Percentage [AWS Metrics]", + "type": "metrics" + } + }, + "coreMigrationVersion": "7.15.0", + "id": "aws-c4a82470-42aa-11ec-a440-675e476c18bb", + "migrationVersion": { + "visualization": "7.14.0" + }, + "references": [ + { + "id": "metrics-*", + "name": "metrics_0_index_pattern", + "type": "index-pattern" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/aws/kibana/visualization/aws-c631dc40-42ab-11ec-a440-675e476c18bb.json b/test/packages/aws/kibana/visualization/aws-c631dc40-42ab-11ec-a440-675e476c18bb.json new file mode 100644 index 0000000000..388b75dc8a --- /dev/null +++ b/test/packages/aws/kibana/visualization/aws-c631dc40-42ab-11ec-a440-675e476c18bb.json @@ -0,0 +1,127 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "S3 Storage Lens Encrypted Bytes Percentage [AWS Metrics]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "background_color_rules": [ + { + "id": "8d0ce9b0-42aa-11ec-93b6-03cae606ce61" + } + ], + "bar_color_rules": [ + { + "id": "ddaf4ed0-42aa-11ec-a5a1-73d3b46a933e" + } + ], + "drop_last_bucket": 0, + "filter": { + "language": "kuery", + "query": "aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" " + }, + "gauge_color_rules": [ + { + "id": "0d38a660-42ab-11ec-a5a1-73d3b46a933e" + } + ], + "gauge_inner_width": 10, + "gauge_style": "half", + "gauge_width": 10, + "id": "7c0ba441-be60-42ec-9d68-6959f1c1e709", + "index_pattern_ref_name": "metrics_0_index_pattern", + "interval": "", + "isModelInvalid": false, + "markdown": "test\t{{ object_lock_bytes_percentage.last.raw }}", + "max_lines_legend": 1, + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "rgba(96,146,192,1)", + "fill": 0.5, + "formatter": "percent", + "id": "4e7561ec-a29c-4558-9d40-d8f378c85daf", + "label": "Encrypted", + "line_width": 1, + "metrics": [ + { + "field": "aws.s3_storage_lens.metrics.EncryptedStorageBytes.avg", + "id": "5e067d85-0dfc-4208-a72b-75279b58b2b2", + "type": "sum" + }, + { + "field": "aws.s3_storage_lens.metrics.StorageBytes.avg", + "id": "66645e10-42aa-11ec-93b6-03cae606ce61", + "type": "sum" + }, + { + "id": "707b8720-42aa-11ec-93b6-03cae606ce61", + "script": "divide(params.Encrypted, params.StorageBytes)", + "type": "math", + "variables": [ + { + "field": "5e067d85-0dfc-4208-a72b-75279b58b2b2", + "id": "73b2bd50-42aa-11ec-93b6-03cae606ce61", + "name": "Encrypted" + }, + { + "field": "66645e10-42aa-11ec-93b6-03cae606ce61", + "id": "7c32cba0-42aa-11ec-93b6-03cae606ce61", + "name": "StorageBytes" + } + ] + } + ], + "palette": { + "name": "default", + "type": "palette" + }, + "point_size": 1, + "separate_axis": 0, + "split_mode": "everything", + "stacked": "none", + "type": "timeseries" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "", + "time_range_mode": "entire_time_range", + "tooltip_mode": "show_all", + "truncate_legend": 1, + "type": "metric", + "use_kibana_indexes": true + }, + "title": "S3 Storage Lens Encrypted Bytes Percentage [AWS Metrics]", + "type": "metrics" + } + }, + "coreMigrationVersion": "7.15.0", + "id": "aws-c631dc40-42ab-11ec-a440-675e476c18bb", + "migrationVersion": { + "visualization": "7.14.0" + }, + "references": [ + { + "id": "metrics-*", + "name": "metrics_0_index_pattern", + "type": "index-pattern" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/aws/kibana/visualization/aws-e1c345e0-42a9-11ec-a440-675e476c18bb.json b/test/packages/aws/kibana/visualization/aws-e1c345e0-42a9-11ec-a440-675e476c18bb.json new file mode 100644 index 0000000000..bfbe5e2631 --- /dev/null +++ b/test/packages/aws/kibana/visualization/aws-e1c345e0-42a9-11ec-a440-675e476c18bb.json @@ -0,0 +1,111 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "S3 Storage Lens Total Storage Class Distribution [AWS Metrics]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "background_color_rules": [ + { + "id": "88f52970-42a8-11ec-afe1-bb42ec488e9b" + } + ], + "bar_color_rules": [ + { + "id": "5fac2960-42a8-11ec-afe1-bb42ec488e9b" + } + ], + "drop_last_bucket": 0, + "filter": { + "language": "kuery", + "query": "" + }, + "gauge_color_rules": [ + { + "id": "864db020-42a8-11ec-afe1-bb42ec488e9b" + } + ], + "gauge_inner_width": 10, + "gauge_style": "half", + "gauge_width": 10, + "id": "c08f117f-f0e8-4d10-ace7-5d2a5aab35c0", + "index_pattern_ref_name": "metrics_0_index_pattern", + "interval": "3d", + "isModelInvalid": false, + "max_lines_legend": 1, + "series": [ + { + "axis_position": "left", + "chart_type": "bar", + "color": "rgba(96,146,192,1)", + "fill": "1", + "filter": { + "language": "kuery", + "query": "aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\"" + }, + "formatter": "bytes", + "id": "de4a05bf-2a2d-422d-98ba-49350f7442d9", + "label": "Total Storage", + "line_width": 1, + "metrics": [ + { + "field": "aws.s3_storage_lens.metrics.StorageBytes.avg", + "id": "cf9f8d20-f581-4036-8f5f-07748b6f9533", + "type": "sum" + } + ], + "palette": { + "name": "default", + "type": "palette" + }, + "point_size": 1, + "separate_axis": 1, + "split_mode": "terms", + "stacked": "none", + "terms_field": "aws.dimensions.storage_class", + "terms_order_by": "cf9f8d20-f581-4036-8f5f-07748b6f9533", + "terms_size": "5", + "type": "timeseries" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "", + "time_range_mode": "last_value", + "tooltip_mode": "show_all", + "truncate_legend": 1, + "type": "top_n", + "use_kibana_indexes": true + }, + "title": "S3 Storage Lens Total Storage Class Distribution [AWS Metrics]", + "type": "metrics" + } + }, + "coreMigrationVersion": "7.15.0", + "id": "aws-e1c345e0-42a9-11ec-a440-675e476c18bb", + "migrationVersion": { + "visualization": "7.14.0" + }, + "references": [ + { + "id": "metrics-*", + "name": "metrics_0_index_pattern", + "type": "index-pattern" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/aws/kibana/visualization/aws-ef9717b0-427b-11ec-9b39-b9df88395505.json b/test/packages/aws/kibana/visualization/aws-ef9717b0-427b-11ec-9b39-b9df88395505.json new file mode 100644 index 0000000000..3660b7db19 --- /dev/null +++ b/test/packages/aws/kibana/visualization/aws-ef9717b0-427b-11ec-9b39-b9df88395505.json @@ -0,0 +1,34 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "S3 Storage Lens Data Protection Markdown [AWS Metrics]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "fontSize": 14, + "markdown": "Data Protection", + "openLinksInNewTab": false + }, + "title": "S3 Storage Lens Data Protection Markdown [AWS Metrics]", + "type": "markdown" + } + }, + "coreMigrationVersion": "7.15.0", + "id": "aws-ef9717b0-427b-11ec-9b39-b9df88395505", + "migrationVersion": { + "visualization": "7.14.0" + }, + "references": [], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/aws/kibana/visualization/aws-f9b2beb0-42a9-11ec-a440-675e476c18bb.json b/test/packages/aws/kibana/visualization/aws-f9b2beb0-42a9-11ec-a440-675e476c18bb.json new file mode 100644 index 0000000000..9dd5fc35b4 --- /dev/null +++ b/test/packages/aws/kibana/visualization/aws-f9b2beb0-42a9-11ec-a440-675e476c18bb.json @@ -0,0 +1,107 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "S3 Storage Lens Object Count Class Distribution [AWS Metrics]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "background_color_rules": [ + { + "id": "88f52970-42a8-11ec-afe1-bb42ec488e9b" + } + ], + "bar_color_rules": [ + { + "id": "5fac2960-42a8-11ec-afe1-bb42ec488e9b" + } + ], + "drop_last_bucket": 0, + "filter": { + "language": "kuery", + "query": "aws.dimensions.record_type : \"ACCOUNT\" and not aws.dimensions.storage_class : \"-\" " + }, + "gauge_color_rules": [ + { + "id": "864db020-42a8-11ec-afe1-bb42ec488e9b" + } + ], + "gauge_inner_width": 10, + "gauge_style": "half", + "gauge_width": 10, + "id": "c08f117f-f0e8-4d10-ace7-5d2a5aab35c0", + "index_pattern_ref_name": "metrics_0_index_pattern", + "interval": "3d", + "isModelInvalid": false, + "max_lines_legend": 1, + "series": [ + { + "axis_position": "left", + "chart_type": "bar", + "color": "rgba(231,102,76,1)", + "fill": "1", + "formatter": "number", + "id": "de4a05bf-2a2d-422d-98ba-49350f7442d9", + "label": "Object Count", + "line_width": 1, + "metrics": [ + { + "field": "aws.s3_storage_lens.metrics.ObjectCount.avg", + "id": "cf9f8d20-f581-4036-8f5f-07748b6f9533", + "type": "sum" + } + ], + "palette": { + "name": "default", + "type": "palette" + }, + "point_size": 1, + "separate_axis": 1, + "split_mode": "terms", + "stacked": "none", + "terms_field": "aws.dimensions.storage_class", + "terms_order_by": "cf9f8d20-f581-4036-8f5f-07748b6f9533", + "terms_size": "5", + "type": "timeseries" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "", + "time_range_mode": "last_value", + "tooltip_mode": "show_all", + "truncate_legend": 1, + "type": "top_n", + "use_kibana_indexes": true + }, + "title": "S3 Storage Lens Object Count Class Distribution [AWS Metrics]", + "type": "metrics" + } + }, + "coreMigrationVersion": "7.15.0", + "id": "aws-f9b2beb0-42a9-11ec-a440-675e476c18bb", + "migrationVersion": { + "visualization": "7.14.0" + }, + "references": [ + { + "id": "metrics-*", + "name": "metrics_0_index_pattern", + "type": "index-pattern" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/aws/manifest.yml b/test/packages/aws/manifest.yml index 3350f8a718..73f660bba9 100644 --- a/test/packages/aws/manifest.yml +++ b/test/packages/aws/manifest.yml @@ -6,16 +6,14 @@ title: AWS # the actual apache package in the registry at any given time. version: 999.999.999 license: basic -description: AWS Integration +description: Collect logs and metrics from Amazon Web Services with Elastic Agent. type: integration categories: - aws - cloud - - network - - security -release: beta +release: ga conditions: - kibana.version: '^7.9.0' + kibana.version: "^7.15.0 || ^8.0.0" screenshots: - src: /img/metricbeat-aws-overview.png title: metricbeat aws overview @@ -33,31 +31,31 @@ vars: multi: false required: false show_user: false - description: Directory of the shared credentials file. + description: Directory of the shared credentials file - name: credential_profile_name type: text title: Credential Profile Name multi: false required: false - show_user: true + show_user: false - name: access_key_id type: text title: Access Key ID multi: false required: false - show_user: false + show_user: true - name: secret_access_key type: text title: Secret Access Key multi: false required: false - show_user: false + show_user: true - name: session_token type: text title: Session Token multi: false required: false - show_user: false + show_user: true - name: role_arn type: text title: Role ARN @@ -71,11 +69,18 @@ vars: required: false show_user: false default: "amazonaws.com" - description: URL of the entry point for an AWS web service. + description: URL of the entry point for an AWS web service + - name: proxy_url + type: text + title: Proxy URL + multi: false + required: false + show_user: false + description: URL to proxy connections in the form of http[s]://:@: policy_templates: - name: billing - title: AWS Billing - description: Collect AWS billing metrics + title: AWS Billing Metrics + description: Collect billing metrics from Amazon Web Services with Elastic Agent data_streams: - billing inputs: @@ -94,30 +99,21 @@ policy_templates: size: 2640x2240 type: image/png - name: cloudtrail - title: AWS Cloudtrail - description: Collect logs from AWS Cloudtrail + title: AWS Cloudtrail Logs + description: Collect and parse logs from AWS Cloudtrail with Elastic Agent data_streams: - cloudtrail + categories: + - security inputs: - - type: s3 + - type: aws-s3 title: Collect logs from Cloudtrail service - description: Collecting Cloudtrail logs using S3 input + description: Collecting Cloudtrail logs using aws-s3 input + input_group: logs + - type: httpjson + title: Collect logs from third-party REST API (experimental) + description: Collect logs from third-party REST API (experimental) input_group: logs - vars: - - name: visibility_timeout - type: text - title: Visibility Timeout - multi: false - required: false - show_user: false - description: The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours. - - name: api_timeout - type: text - title: API Timeout - multi: false - required: false - show_user: false - description: The maximum duration of AWS API can take. The maximum is half of the visibility timeout value. icons: - src: /img/logo_cloudtrail.svg title: AWS Cloudtrail logo @@ -130,66 +126,31 @@ policy_templates: type: image/png - name: cloudwatch title: AWS CloudWatch - description: Collect logs and metrics from CloudWatch + description: Collect logs and metrics from Amazon CloudWatch with Elastic Agent data_streams: - cloudwatch_logs - cloudwatch_metrics inputs: - - type: s3 + - type: aws-s3 title: Collect logs from CloudWatch - description: Collecting logs from CloudWatch using S3 input + description: Collecting logs from CloudWatch using aws-s3 input input_group: logs - vars: - - name: visibility_timeout - type: text - title: Visibility Timeout - multi: false - required: false - show_user: false - description: The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours. - - name: api_timeout - type: text - title: API Timeout - multi: false - required: false - show_user: false - description: The maximum duration of AWS API can take. The maximum is half of the visibility timeout value. - type: aws/metrics title: Collect metrics from CloudWatch description: Collecting metrics from AWS CloudWatch input_group: metrics - vars: - - name: metrics - type: yaml - title: Metrics - multi: false - required: true - show_user: true - default: | - - namespace: AWS/EC2 - resource_type: ec2:instance - name: - - CPUUtilization - - DiskWriteOps - statistic: - - Average - - Maximum - # dimensions: - # - name: InstanceId - # value: i-123456 - # tags: - # - key: created-by - # value: foo icons: - src: /img/logo_cloudwatch.svg title: AWS CloudWatch logo size: 32x32 type: image/svg+xml - name: dynamodb - title: AWS DynamoDB - description: Collect AWS DynamoDB metrics + title: AWS DynamoDB Metrics + description: Collect metrics from Amazon DynamoDB service with Elastic Agent data_streams: - dynamodb + categories: + - datastore inputs: - type: aws/metrics title: Collect dynamodb metrics @@ -206,10 +167,12 @@ policy_templates: size: 2640x2240 type: image/png - name: ebs - title: AWS EBS - description: Collect AWS EBS metrics + title: AWS EBS Metrics + description: Collect metrics from Amazon Elastic Block Storage service with Elastic Agent data_streams: - ebs + categories: + - datastore inputs: - type: aws/metrics title: Collect EBS metrics @@ -227,30 +190,15 @@ policy_templates: type: image/png - name: ec2 title: AWS EC2 - description: Collect logs and metrics from EC2 service + description: Collect logs and metrics from Amazon Elastic Compute Cloud service with Elastic Agent data_streams: - ec2_logs - ec2_metrics inputs: - - type: s3 + - type: aws-s3 title: Collect logs from EC2 service - description: Collecting EC2 logs using S3 input + description: Collecting EC2 logs using aws-s3 input input_group: logs - vars: - - name: visibility_timeout - type: text - title: Visibility Timeout - multi: false - required: false - show_user: false - description: The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours. - - name: api_timeout - type: text - title: API Timeout - multi: false - required: false - show_user: false - description: The maximum duration of AWS API can take. The maximum is half of the visibility timeout value. - type: aws/metrics title: Collect metrics from EC2 service description: Collecting EC2 metrics using AWS CloudWatch @@ -267,30 +215,17 @@ policy_templates: type: image/png - name: elb title: AWS ELB - description: Collect logs and metrics from ELB service + description: Collect logs and metrics from Amazon Elastic Load Balancing service with Elastic Agent data_streams: - elb_logs - elb_metrics + categories: + - network inputs: - - type: s3 + - type: aws-s3 title: Collect logs from ELB service - description: Collecting ELB logs using S3 input + description: Collecting ELB logs using aws-s3 input input_group: logs - vars: - - name: visibility_timeout - type: text - title: Visibility Timeout - multi: false - required: false - show_user: false - description: The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours. - - name: api_timeout - type: text - title: API Timeout - multi: false - required: false - show_user: false - description: The maximum duration of AWS API can take. The maximum is half of the visibility timeout value. - type: aws/metrics title: Collect metrics from ELB service description: Collecting ELB metrics using AWS CloudWatch @@ -310,8 +245,8 @@ policy_templates: size: 1684x897 type: image/png - name: lambda - title: AWS Lambda - description: Collect AWS Lambda metrics + title: AWS Lambda Metrics + description: Collect metrics from AWS Lambda service with Elastic Agent data_streams: - lambda inputs: @@ -330,10 +265,12 @@ policy_templates: size: 2640x2240 type: image/png - name: natgateway - title: AWS NATGateway - description: Collect AWS NATGateway metrics + title: AWS NAT Gateway Metrics + description: Collect metrics from Amazon NAT Gateways with Elastic Agent data_streams: - natgateway + categories: + - network inputs: - type: aws/metrics title: Collect NATGateway metrics @@ -345,10 +282,12 @@ policy_templates: size: 32x32 type: image/svg+xml - name: rds - title: AWS RDS - description: Collect AWS RDS metrics + title: AWS RDS Metrics + description: Collect metrics from Amazon Relational Database Service with Elastic Agent data_streams: - rds + categories: + - datastore inputs: - type: aws/metrics title: Collect RDS metrics @@ -366,31 +305,19 @@ policy_templates: type: image/png - name: s3 title: AWS S3 - description: Collect AWS S3 metrics + description: Collect logs and metrics from Amazon Simple Storage Service with Elastic Agent data_streams: - s3_daily_storage - s3_request - s3access + categories: + - datastore + - security inputs: - - type: s3 + - type: aws-s3 title: Collect S3 access logs - description: Collecting S3 access logs using S3 input + description: Collecting S3 access logs using aws-s3 input input_group: logs - vars: - - name: visibility_timeout - type: text - title: Visibility Timeout - multi: false - required: false - show_user: false - description: The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours. - - name: api_timeout - type: text - title: API Timeout - multi: false - required: false - show_user: false - description: The maximum duration of AWS API can take. The maximum is half of the visibility timeout value. - type: aws/metrics title: Collect metrics from S3 description: Collecting S3 metrics using AWS CloudWatch @@ -409,9 +336,31 @@ policy_templates: title: filebeat aws s3access overview size: 1684x897 type: image/png + - name: s3_storage_lens + title: AWS S3 Storage Lens + description: Collect metrics from AWS S3 Storage Lens with Elastic Agent + data_streams: + - s3_storage_lens + categories: + - datastore + inputs: + - type: aws/metrics + title: Collect metrics from AWS S3 Storage Lens + description: Collecting AWS S3 Storage Lens metrics using AWS CloudWatch + input_group: metrics + icons: + - src: /img/logo_s3_storage_lens.svg + title: AWS S3 storage lens logo + size: 32x32 + type: image/svg+xml + screenshots: + - src: /img/metricbeat-aws-s3-storage-lens-overview.png + title: metricbeat aws s3 storage lens overview + size: 2640x2240 + type: image/png - name: sns - title: AWS SNS - description: Collect AWS SNS metrics + title: AWS SNS Metrics + description: Collect metrics from Amazon Simple Notification Service with Elastic Agent data_streams: - sns inputs: @@ -430,8 +379,8 @@ policy_templates: size: 2640x2240 type: image/png - name: sqs - title: AWS SQS - description: Collect AWS SQS metrics + title: AWS SQS Metrics + description: Collect metrics from Amazon Simple Queue Service with Elastic Agent data_streams: - sqs inputs: @@ -450,10 +399,12 @@ policy_templates: size: 2640x2240 type: image/png - name: transitgateway - title: AWS Transit Gateway - description: Collect AWS Transit Gateway metrics + title: AWS Transit Gateway Metrics + description: Collect metrics from AWS Transit Gateways with Elastic Agent data_streams: - transitgateway + categories: + - network inputs: - type: aws/metrics title: Collect Transit Gateway metrics @@ -465,8 +416,8 @@ policy_templates: size: 32x32 type: image/svg+xml - name: usage - title: AWS Usage - description: Collect AWS Usage metrics + title: AWS Usage Metrics + description: Collect usage metrics from Amazon Web Services with Elastic Agent data_streams: - usage inputs: @@ -480,40 +431,30 @@ policy_templates: size: 2640x2240 type: image/png - name: vpcflow - title: AWS VPC Flow - description: Collect AWS vpcflow logs + title: AWS VPC Flow Logs + description: Collect VPC flow logs from Amazon Web Services with Elastic Agent data_streams: - vpcflow + categories: + - network + - security inputs: - - type: s3 + - type: aws-s3 title: Collect VPC Flow logs - description: Collecting VPC Flow logs using S3 input + description: Collecting VPC Flow logs using aws-s3 input input_group: logs - vars: - - name: visibility_timeout - type: text - title: Visibility Timeout - multi: false - required: false - show_user: false - description: The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours. - - name: api_timeout - type: text - title: API Timeout - multi: false - required: false - show_user: false - description: The maximum duration of AWS API can take. The maximum is half of the visibility timeout value. icons: - src: /img/logo_vpcflow.svg title: AWS VPC logo size: 32x32 type: image/svg+xml - name: vpn - title: AWS VPN - description: Collect AWS VPN metrics + title: AWS VPN Metrics + description: Collect VPN metrics from Amazon Web Services with Elastic Agent data_streams: - vpn + categories: + - network inputs: - type: aws/metrics title: Collect VPN metrics @@ -524,5 +465,23 @@ policy_templates: title: AWS VPN logo size: 32x32 type: image/svg+xml + - name: waf + title: AWS WAF Logs + description: Collect AWS WAF logs + data_streams: + - waf + categories: + - network + - security + inputs: + - type: aws-s3 + title: Collect WAF logs + description: Collecting WAF logs using aws-s3 input + input_group: logs + icons: + - src: /img/logo_waf.svg + title: AWS VPC logo + size: 32x32 + type: image/svg+xml owner: - github: elastic/integrations-platforms + github: elastic/integrations diff --git a/test/packages/kubernetes/_dev/build/build.yml b/test/packages/kubernetes/_dev/build/build.yml index 002aa15659..08d85edcf9 100644 --- a/test/packages/kubernetes/_dev/build/build.yml +++ b/test/packages/kubernetes/_dev/build/build.yml @@ -1,3 +1,3 @@ dependencies: ecs: - reference: git@1.10 + reference: git@1.12 diff --git a/test/packages/kubernetes/_dev/build/docs/README.md b/test/packages/kubernetes/_dev/build/docs/README.md index 5a2d112c79..8713649963 100644 --- a/test/packages/kubernetes/_dev/build/docs/README.md +++ b/test/packages/kubernetes/_dev/build/docs/README.md @@ -1,4 +1,93 @@ # Kubernetes integration -This integration is used to collect metrics from -[Kubernetes clusters](https://kubernetes.io/). \ No newline at end of file +This integration is used to collect logs and metrics from +[Kubernetes clusters](https://kubernetes.io/). + +As one of the main pieces provided for Kubernetes monitoring, this integration is capable of fetching metrics from several components: + +- [kubelet](https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/) +- [kube-state-metrics](https://github.com/kubernetes/kube-state-metrics) +- [apiserver](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/) +- [controller-manager](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-controller-manager/) +- [scheduler](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-scheduler/) +- [proxy](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-proxy/) + +Some of the previous components are running on each of the Kubernetes nodes (like `kubelet` or `proxy`) while others provide +a single cluster-wide endpoint. This is important to determine the optimal configuration and running strategy +for the different datasets included in the integration. + + +#### Kubernetes endpoints and metricsets + +Kubernetes module is a bit complex as its internal datasets require access to a wide variety of endpoints. + +This section highlights and introduces some groups of datasets with similar endpoint access needs. +For more details on the datasets see `configuration example` and the `datasets` sections below. + + +#### node / system / pod / container / module / volume + +The datasets `container`, `node`, `pod`, `system` and `volume` require access to the `kubelet endpoint` in each of +the Kubernetes nodes, hence it's recommended to include them as part +of an `Agent DaemonSet` or standalone Agents running on the hosts. + +Depending on the version and configuration of Kubernetes nodes, `kubelet` might provide a read only http port (typically 10255), +which is used in some configuration examples. But in general, and lately, this endpoint requires SSL (`https`) access +(to port 10250 by default) and token based authentication. + + +##### state_* and event + +All datasets with the `state_` prefix require `hosts` field pointing to `kube-state-metrics` +service within the cluster. As the service provides cluster-wide metrics, there's no need to fetch them per node, +hence the recommendation is to run these datasets as part of an `Agent Deployment` with one only replica. + +Note: Kube-state-metrics is not deployed by default in Kubernetes. For these cases the instructions for its +deployment are available [here](https://github.com/kubernetes/kube-state-metrics#kubernetes-deployment). +Generally `kube-state-metrics` runs a `Deployment` and is accessible via a service called `kube-state-metrics` on +`kube-system` namespace, which will be the service to use in our configuration. + +state_* datasets are not enabled by default. + +#### apiserver + +The apiserver dataset requires access to the Kubernetes API, which should be easily available in all Kubernetes +environments. Depending on the Kubernetes configuration, the API access might require SSL (`https`) and token +based authentication. + +#### proxy + +The proxy dataset requires access to the proxy endpoint in each of Kubernetes nodes, hence it's recommended +to configure it as a part of an `Agent DaemonSet`. + +#### scheduler and controllermanager + +These datasets require access to the Kubernetes `controller-manager` and `scheduler` endpoints. By default, these pods +run only on master nodes, and they are not exposed via a Service, but there are different strategies +available for its configuration: + +- Create `Kubernetes Services` to make `kube-controller-manager` and `kube-scheduler` available and configure + the datasets to point to these services as part of an `Agent Deployment`. +- Run these datasets as part an `Agent Daemonset` (with HostNetwork setting) with a `nodeSelector` to only run on Master nodes. + +These datasets are not enabled by default. + +Note: In some "As a Service" Kubernetes implementations, like `GKE`, the master nodes or even the pods running on +the masters won't be visible. In these cases it won't be possible to use `scheduler` and `controllermanager` metricsets. + +#### container-logs + +The container-logs dataset requires access to the log files in each Kubernetes node where the container logs are stored. +This defaults to `/var/log/containers/*${kubernetes.container.id}.log`. + +## Compatibility + +The Kubernetes package is tested with Kubernetes 1.13.x, 1.14.x, 1.15.x, 1.16.x, 1.17.x, and 1.18.x + +## Dashboard + +Kubernetes integration is shipped including default dashboards for `apiserver`, `controllermanager`, `overview`, `proxy` and `scheduler`. + +If you are using HA for those components, be aware that when gathering data from all instances the dashboard will usually show the average of the metrics. For those scenarios filtering by hosts or service address is possible. + +Cluster selector in `overview` dashboard helps in distinguishing and filtering metrics collected from multiple clusters. If you want to focus on a subset of the Kubernetes clusters for monitoring a specific scenario, this cluster selector could be a handy tool. Note that this selector gets populated from the `orchestrator.cluster.name` field that may not always be available. This field gets its value from sources like `kube_config`, `kubeadm-config` configMap, and Google Cloud's meta API for GKE. If the sources mentioned above don't provide this value, metricbeat will not report it. However, you can always use [processors](https://www.elastic.co/guide/en/beats/metricbeat/current/defining-processors.html) to set this field and utilize it in the `cluster overview` dashboard. diff --git a/test/packages/kubernetes/_dev/build/docs/container-logs.md b/test/packages/kubernetes/_dev/build/docs/container-logs.md new file mode 100644 index 0000000000..3595e1c268 --- /dev/null +++ b/test/packages/kubernetes/_dev/build/docs/container-logs.md @@ -0,0 +1,6 @@ +# container-logs + +container-logs integration collects and parses logs of Kubernetes containers. + +It requires access to the log files in each Kubernetes node where the container logs are stored. +This defaults to `/var/log/containers/*${kubernetes.container.id}.log`. \ No newline at end of file diff --git a/test/packages/kubernetes/_dev/build/docs/events.md b/test/packages/kubernetes/_dev/build/docs/events.md new file mode 100644 index 0000000000..bd2ce5a228 --- /dev/null +++ b/test/packages/kubernetes/_dev/build/docs/events.md @@ -0,0 +1,16 @@ +# events + +## Metrics + +### event + +This is the `event` dataset of the Kubernetes package. It collects Kubernetes events +related metrics. + +If Leader Election is activated (default behaviour) only the `elastic agent` which holds the leadership lock +will retrieve events related metrics. +This is relevant in multi-node kubernetes cluster and prevents duplicate data. + +{{event "event"}} + +{{fields "event"}} \ No newline at end of file diff --git a/test/packages/kubernetes/_dev/build/docs/kube-apiserver.md b/test/packages/kubernetes/_dev/build/docs/kube-apiserver.md new file mode 100644 index 0000000000..a28c98cc00 --- /dev/null +++ b/test/packages/kubernetes/_dev/build/docs/kube-apiserver.md @@ -0,0 +1,37 @@ +# kube-apiserver + +## Metrics + +### apiserver + +This is the `apiserver` dataset of the Kubernetes package, in charge of retrieving metrics +from the Kubernetes API (available at `/metrics`). + +This metricset needs access to the `apiserver` component of Kubernetes, accessible typically +by any POD via the `kubernetes.default` service or via environment +variables (`KUBERNETES_SERVICE_HOST` and `KUBERNETES_SERVICE_PORT`). + +If Leader Election is activated (default behaviour) only the `elastic agent` which holds the leadership lock +will retrieve metrics from the `apiserver`. +This is relevant in multi-node kubernetes cluster and prevents duplicate data. + +When the API uses https, the pod will need to authenticate using its default token and trust +the server using the appropriate CA file. + +Configuration example using https and token based authentication: + + +In order to access the `/metrics` path of the API service, some Kubernetes environments might +require the following permission to be added to a ClusterRole. + +```yaml +rules: +- nonResourceURLs: + - /metrics + verbs: + - get +``` + +{{event "apiserver"}} + +{{fields "apiserver"}} \ No newline at end of file diff --git a/test/packages/kubernetes/_dev/build/docs/kube-controller-manager.md b/test/packages/kubernetes/_dev/build/docs/kube-controller-manager.md new file mode 100644 index 0000000000..01faeab816 --- /dev/null +++ b/test/packages/kubernetes/_dev/build/docs/kube-controller-manager.md @@ -0,0 +1,12 @@ +# kube-controller-manager + +## Metrics + +### controllermanager + +This is the `controllermanager` dataset for the Kubernetes package. It collects from +Kubernetes controller component `metrics` endpoint. + +{{event "controllermanager"}} + +{{fields "controllermanager"}} \ No newline at end of file diff --git a/test/packages/kubernetes/_dev/build/docs/kube-proxy.md b/test/packages/kubernetes/_dev/build/docs/kube-proxy.md new file mode 100644 index 0000000000..6df5baf0c8 --- /dev/null +++ b/test/packages/kubernetes/_dev/build/docs/kube-proxy.md @@ -0,0 +1,12 @@ +# kube-proxy + +## Metrics + +### proxy + +This is the `proxy` dataset of the Kubernetes package. It collects metrics +from Kubernetes Proxy component. + +{{event "proxy"}} + +{{fields "proxy"}} \ No newline at end of file diff --git a/test/packages/kubernetes/_dev/build/docs/kube-scheduler.md b/test/packages/kubernetes/_dev/build/docs/kube-scheduler.md new file mode 100644 index 0000000000..bfd8b65c25 --- /dev/null +++ b/test/packages/kubernetes/_dev/build/docs/kube-scheduler.md @@ -0,0 +1,12 @@ +# kube-scheduler + +## Metrics + +### scheduler + +This is the `scheduler` dataset of the Kubernetes package. It collects metrics +from Kubernetes Scheduler component. + +{{event "scheduler"}} + +{{fields "scheduler"}} \ No newline at end of file diff --git a/test/packages/kubernetes/_dev/build/docs/kube-state-metrics.md b/test/packages/kubernetes/_dev/build/docs/kube-state-metrics.md new file mode 100644 index 0000000000..4358201cd3 --- /dev/null +++ b/test/packages/kubernetes/_dev/build/docs/kube-state-metrics.md @@ -0,0 +1,132 @@ +# kube-state-metrics + +## Metrics + +If Leader Election is activated (default behaviour) only the `elastic agent` which holds the leadership lock +will retrieve metrics from the `kube_state_metrics`. +This is relevant in multi-node kubernetes cluster and prevents duplicate data. + +### state_container + +This is the `state_container` dataset of the Kubernetes package. It collects container related +metrics from `kube_state_metrics`. + +{{event "state_container"}} + +{{fields "state_container"}} + +### state_cronjob + +This is the `state_cronjob` dataset of the Kubernetes package. It collects cronjob related +metrics from `kube_state_metrics`. + +{{event "state_cronjob"}} + +{{fields "state_cronjob"}} + +### state_daemonset + +This is the `state_daemonset` dataset of the Kubernetes package. It collects daemonset related +metrics from `kube_state_metrics`. + +{{event "state_daemonset"}} + +{{fields "state_daemonset"}} + +### state_deployment + +This is the `state_deployment` dataset of the Kubernetes package. It collects deployment related +metrics from `kube_state_metrics`. + +{{event "state_deployment"}} + +{{fields "state_deployment"}} + +### state_job + +This is the `state_job` dataset of the Kubernetes package. It collects job related +metrics from `kube_state_metrics`. + +{{event "state_job"}} + +{{fields "state_job"}} + +### state_node + +This is the `state_node` dataset of the Kubernetes package. It collects node related +metrics from `kube_state_metrics`. + +{{event "state_node"}} + +{{fields "state_node"}} + +### state_persistentvolume + +This is the `state_persistentvolume` dataset of the Kubernetes package. It collects +PersistentVolume related metrics from `kube_state_metrics`. + +{{event "state_persistentvolume"}} + +{{fields "state_persistentvolume"}} + +### state_persistentvolumeclaim + +This is the `state_persistentvolumeclaim` dataset of the Kubernetes package. It collects +PersistentVolumeClaim related metrics from `kube_state_metrics`. + +{{event "state_persistentvolumeclaim"}} + +{{fields "state_persistentvolumeclaim"}} + +### state_pod + +This is the `state_pod` dataset of the Kubernetes package. It collects +Pod related metrics from `kube_state_metrics`. + +{{event "state_pod"}} + +{{fields "state_pod"}} + +### state_replicaset + +This is the `state_replicaset` dataset of the Kubernetes package. It collects +Replicaset related metrics from `kube_state_metrics`. + +{{event "state_replicaset"}} + +{{fields "state_replicaset"}} + +### state_resourcequota + +This is the `state_resourcequota` dataset of the Kubernetes package. It collects ResourceQuota related metrics +from `kube_state_metrics`. + +{{event "state_resourcequota"}} + +{{fields "state_resourcequota"}} + +### state_service + +This is the `state_service` dataset of the Kubernetes package. It collects +Service related metrics from `kube_state_metrics`. + +{{event "state_service"}} + +{{fields "state_service"}} + +### state_statefulset + +This is the `state_statefulset` dataset of the Kubernetes package. + +{{event "state_statefulset"}} + +{{fields "state_statefulset"}} + +### state_storageclass + +This is the `state_storageclass` dataset of the Kubernetes package. It collects +StorageClass related metrics from `kube_state_metrics`. + +{{event "state_storageclass"}} + +{{fields "state_storageclass"}} \ No newline at end of file diff --git a/test/packages/kubernetes/_dev/build/docs/kubelet.md b/test/packages/kubernetes/_dev/build/docs/kubelet.md new file mode 100644 index 0000000000..54e684a860 --- /dev/null +++ b/test/packages/kubernetes/_dev/build/docs/kubelet.md @@ -0,0 +1,48 @@ +# kubelet + +## Metrics + +### container + +This is the `container` dataset of the Kubernetes package. It collects container related metrics +from Kubelet's monitoring APIs. + +{{event "container"}} + +{{fields "container"}} + +### node + +This is the `node` dataset of the Kubernetes package. It collects Node related metrics +from Kubelet's monitoring APIs. + +{{event "node"}} + +{{fields "node"}} + +### pod + +This is the `pod` dataset of the Kubernetes package. It collects Pod related metrics +from Kubelet's monitoring APIs. + +{{event "pod"}} + +{{fields "pod"}} + +### system + +This is the `system` dataset of the Kubernetes package. It collects System related metrics +from Kubelet's monitoring APIs. + +{{event "system"}} + +{{fields "system"}} + +### volume + +This is the `volume` dataset of the Kubernetes package. It collects Volume related metrics +from Kubelet's monitoring APIs. + +{{event "volume"}} + +{{fields "volume"}} \ No newline at end of file diff --git a/test/packages/kubernetes/_dev/deploy/k8s/conrjob.yaml b/test/packages/kubernetes/_dev/deploy/k8s/conrjob.yaml new file mode 100644 index 0000000000..472da4598b --- /dev/null +++ b/test/packages/kubernetes/_dev/deploy/k8s/conrjob.yaml @@ -0,0 +1,19 @@ +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: hello +spec: + schedule: "*/1 * * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: hello + image: busybox + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + - date; echo Hello from the Kubernetes cluster + restartPolicy: OnFailure diff --git a/test/packages/kubernetes/_dev/deploy/k8s/example-redis-config.yaml b/test/packages/kubernetes/_dev/deploy/k8s/example-redis-config.yaml new file mode 100644 index 0000000000..7ee3e40d5c --- /dev/null +++ b/test/packages/kubernetes/_dev/deploy/k8s/example-redis-config.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: example-redis-config +data: + redis-config: "" diff --git a/test/packages/kubernetes/_dev/deploy/k8s/job.yaml b/test/packages/kubernetes/_dev/deploy/k8s/job.yaml new file mode 100644 index 0000000000..c1087e28c4 --- /dev/null +++ b/test/packages/kubernetes/_dev/deploy/k8s/job.yaml @@ -0,0 +1,17 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: hello +spec: + template: + spec: + containers: + - name: hello + image: busybox + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + - date; echo Hello from the Kubernetes cluster + restartPolicy: OnFailure + backoffLimit: 4 diff --git a/test/packages/kubernetes/_dev/deploy/k8s/pv.yaml b/test/packages/kubernetes/_dev/deploy/k8s/pv.yaml new file mode 100644 index 0000000000..bca6cd1eb5 --- /dev/null +++ b/test/packages/kubernetes/_dev/deploy/k8s/pv.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: task-pv-volume + labels: + name: task-pv-volume +spec: + storageClassName: generic + capacity: + storage: 2Ki + accessModes: + - ReadWriteOnce + hostPath: + path: "/tmp/data" diff --git a/test/packages/kubernetes/_dev/deploy/k8s/pvc.yaml b/test/packages/kubernetes/_dev/deploy/k8s/pvc.yaml new file mode 100644 index 0000000000..7b24222960 --- /dev/null +++ b/test/packages/kubernetes/_dev/deploy/k8s/pvc.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: task-pv-claim +spec: + storageClassName: generic + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Ki + selector: + matchLabels: + name: task-pv-volume diff --git a/test/packages/kubernetes/_dev/deploy/k8s/rq.yaml b/test/packages/kubernetes/_dev/deploy/k8s/rq.yaml new file mode 100644 index 0000000000..07c6d882a9 --- /dev/null +++ b/test/packages/kubernetes/_dev/deploy/k8s/rq.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: List +items: + - apiVersion: v1 + kind: ResourceQuota + metadata: + name: pods-high + spec: + hard: + cpu: "1000" + memory: 200Ki + pods: "10" + scopeSelector: + matchExpressions: + - operator: In + scopeName: PriorityClass + values: ["high"] diff --git a/test/packages/kubernetes/_dev/deploy/k8s/ss.yaml b/test/packages/kubernetes/_dev/deploy/k8s/ss.yaml new file mode 100644 index 0000000000..037d913837 --- /dev/null +++ b/test/packages/kubernetes/_dev/deploy/k8s/ss.yaml @@ -0,0 +1,22 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: web +spec: + selector: + matchLabels: + app: nginx # has to match .spec.template.metadata.labels + serviceName: "nginx" + replicas: 1 # by default is 1 + template: + metadata: + labels: + app: nginx # has to match .spec.selector.matchLabels + spec: + terminationGracePeriodSeconds: 10 + containers: + - name: nginx + image: k8s.gcr.io/nginx-slim:0.8 + ports: + - containerPort: 80 + name: web diff --git a/test/packages/kubernetes/changelog.yml b/test/packages/kubernetes/changelog.yml index 74b56f0fd5..448c3d2cdd 100644 --- a/test/packages/kubernetes/changelog.yml +++ b/test/packages/kubernetes/changelog.yml @@ -1,9 +1,94 @@ # newer versions go on top - version: "999.999.999" changes: - - description: Give the package highest priority - type: enhancement # can be one of: enhancement, bugfix, breaking-change - link: https://github.com/elastic/elastic-package/pull/460 + - description: Revert Kubernetes namespace field breaking change + type: enhancement + link: https://github.com/elastic/integrations/pull/2209 +- version: "1.4.2" + changes: + - description: Add dimension fields + type: enhancement + link: https://github.com/elastic/integrations/pull/2076 +- version: "1.4.1" + changes: + - description: Remove overriding of index pattern on the Kubernetes overview dashboard + type: enhancement + link: https://github.com/elastic/integrations/pull/2151 +- version: "1.4.0" + changes: + - description: Use filestream input for container_logs data stream + type: enhancement + link: https://github.com/elastic/integrations/pull/2139 +- version: "1.3.3" + changes: + - description: Fix conditions of data_streams that are based on k8s labels & add condition in pipelines + type: bugfix + link: https://github.com/elastic/integrations/pull/2041 +- version: "1.3.2" + changes: + - description: Set default host for proxy to localhost + type: enhancement + link: https://github.com/elastic/integrations/pull/2057 +- version: "1.3.1" + changes: + - description: Uniform with guidelines + type: enhancement + link: https://github.com/elastic/integrations/pull/2057 +- version: "1.3.0" + changes: + - description: Add container_logs ecs fields + type: enhancement + link: https://github.com/elastic/integrations/pull/2033 +- version: "1.2.1" + changes: + - description: Update Kubernetes cluster_ip field type + type: bugfix + link: https://github.com/elastic/integrations/pull/2031 +- version: "1.2.0" + changes: + - description: Update Kubernetes namespace field + type: enhancement + link: https://github.com/elastic/integrations/pull/1792 +- version: "1.1.1" + changes: + - description: Update Kubernetes integration Readme + type: bugfix + link: https://github.com/elastic/integrations/pull/1890 +- version: "1.1.0" + changes: + - description: Update to ECS 1.12.0 + type: enhancement + link: https://github.com/elastic/integrations/pull/1696 +- version: "1.0.0" + changes: + - description: Release Kubernetes as GA + type: enhancement + link: https://github.com/elastic/integrations/pull/1610 +- version: "0.14.1" + changes: + - description: Update default host in kubernetes proxy data stream in kubernetes integration + type: enhancement + link: https://github.com/elastic/integrations/pull/1565 +- version: "0.14.0" + changes: + - description: Add new container logs data stream in kubernetes integration + type: enhancement + link: https://github.com/elastic/integrations/pull/1324 +- version: "0.13.0" + changes: + - description: Leverage dynamic kubernetes provider for controller and scheduler datastream + type: enhancement + link: https://github.com/elastic/integrations/pull/1324 +- version: "0.12.2" + changes: + - description: Add missing field "kubernetes.daemonset.name" field for pod and container data streams + type: bugfix + link: https://github.com/elastic/integrations/pull/1459 +- version: "0.12.1" + changes: + - description: Add missing cluster filter for "orchestrator.cluster.name" field in [Metrics Kubernetes] Overview dashboard and Dashboard section in the integration overview page + type: bugfix + link: https://github.com/elastic/integrations/pull/1413 - version: "0.12.0" changes: - description: Update kubernetes package ecs fields with orchestrator.cluster.url and orchestrator.cluster.name diff --git a/test/packages/kubernetes/data_stream/apiserver/_dev/deploy/k8s/.empty b/test/packages/kubernetes/data_stream/apiserver/_dev/deploy/k8s/.empty new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/packages/kubernetes/data_stream/apiserver/_dev/test/system/test-default-config.yml b/test/packages/kubernetes/data_stream/apiserver/_dev/test/system/test-default-config.yml new file mode 100644 index 0000000000..cf6f20d0c0 --- /dev/null +++ b/test/packages/kubernetes/data_stream/apiserver/_dev/test/system/test-default-config.yml @@ -0,0 +1,7 @@ +service: kubernetes +data_stream: + vars: + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + period: 5s + hosts: + - https://kubernetes.default:443 diff --git a/test/packages/kubernetes/data_stream/apiserver/agent/stream/stream.yml.hbs b/test/packages/kubernetes/data_stream/apiserver/agent/stream/stream.yml.hbs new file mode 100644 index 0000000000..46426492a2 --- /dev/null +++ b/test/packages/kubernetes/data_stream/apiserver/agent/stream/stream.yml.hbs @@ -0,0 +1,18 @@ +metricsets: ["apiserver"] +hosts: +{{#each hosts}} + - {{this}} +{{/each}} +period: {{period}} + +{{#if leaderelection}} +condition: ${kubernetes_leaderelection.leader} == true +{{/if}} + +{{#if bearer_token_file}} +bearer_token_file: {{bearer_token_file}} +ssl.certificate_authorities: +{{#each ssl.certificate_authorities}} + - {{this}} +{{/each}} +{{/if}} diff --git a/test/packages/kubernetes/data_stream/apiserver/fields/agent.yml b/test/packages/kubernetes/data_stream/apiserver/fields/agent.yml new file mode 100644 index 0000000000..da4e652c53 --- /dev/null +++ b/test/packages/kubernetes/data_stream/apiserver/fields/agent.yml @@ -0,0 +1,198 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + diff --git a/test/packages/nginx/data_stream/ingress_controller/fields/base-fields.yml b/test/packages/kubernetes/data_stream/apiserver/fields/base-fields.yml similarity index 65% rename from test/packages/nginx/data_stream/ingress_controller/fields/base-fields.yml rename to test/packages/kubernetes/data_stream/apiserver/fields/base-fields.yml index 2e8955095b..7c798f4534 100644 --- a/test/packages/nginx/data_stream/ingress_controller/fields/base-fields.yml +++ b/test/packages/kubernetes/data_stream/apiserver/fields/base-fields.yml @@ -1,13 +1,12 @@ - name: data_stream.type - external: ecs + type: constant_keyword description: Data stream type. - value: logs - name: data_stream.dataset - external: ecs + type: constant_keyword description: Data stream dataset. - value: nginx.ingress_controller - name: data_stream.namespace - external: ecs + type: constant_keyword + description: Data stream namespace. - name: '@timestamp' type: date description: Event timestamp. diff --git a/test/packages/kubernetes/data_stream/apiserver/fields/ecs.yml b/test/packages/kubernetes/data_stream/apiserver/fields/ecs.yml new file mode 100644 index 0000000000..cd4e3a89b1 --- /dev/null +++ b/test/packages/kubernetes/data_stream/apiserver/fields/ecs.yml @@ -0,0 +1,10 @@ +- external: ecs + name: ecs.version +- external: ecs + name: service.address +- external: ecs + name: service.type +- external: ecs + name: orchestrator.cluster.name +- external: ecs + name: orchestrator.cluster.url diff --git a/test/packages/kubernetes/data_stream/apiserver/fields/fields.yml b/test/packages/kubernetes/data_stream/apiserver/fields/fields.yml new file mode 100644 index 0000000000..8ec75318fb --- /dev/null +++ b/test/packages/kubernetes/data_stream/apiserver/fields/fields.yml @@ -0,0 +1,203 @@ +- name: kubernetes.apiserver + type: group + fields: + - name: request.client + dimension: true + type: keyword + description: | + Client executing requests + - name: request.resource + dimension: true + type: keyword + description: | + Requested resource + - name: request.subresource + dimension: true + type: keyword + description: | + Requested subresource + - name: request.scope + dimension: true + type: keyword + description: | + Request scope (cluster, namespace, resource) + - name: request.verb + dimension: true + type: keyword + description: | + HTTP verb + - name: request.code + dimension: true + type: keyword + description: | + HTTP code + - name: request.content_type + type: keyword + description: | + Request HTTP content type + - name: request.dry_run + type: keyword + description: | + Wether the request uses dry run + - name: request.kind + dimension: true + type: keyword + description: | + Kind of request + - name: request.component + dimension: true + type: keyword + description: | + Component handling the request + - name: request.group + dimension: true + type: keyword + description: | + API group for the resource + - name: request.version + dimension: true + type: keyword + description: | + version for the group + - name: request.handler + dimension: true + type: keyword + description: | + Request handler + - name: request.method + dimension: true + type: keyword + description: | + HTTP method + - name: request.host + dimension: true + type: keyword + description: | + Request host + - name: process + type: group + fields: + - name: cpu.sec + type: double + metric_type: counter + description: CPU seconds + - name: memory.resident.bytes + type: long + format: bytes + unit: byte + metric_type: gauge + description: Bytes in resident memory + - name: memory.virtual.bytes + type: long + format: bytes + unit: byte + metric_type: gauge + description: Bytes in virtual memory + - name: fds.open.count + type: long + metric_type: gauge + description: Number of open file descriptors + - name: started.sec + type: double + metric_type: gauge + description: Seconds since the process started + - name: http + type: group + fields: + - name: request.duration.us.percentile.* + type: object + description: Request duration microseconds percentiles + - name: request.duration.us.sum + type: double + metric_type: counter + unit: micros + description: Request duration microseconds cumulative sum + - name: request.duration.us.count + type: long + metric_type: counter + unit: micros + description: Request count for duration + - name: request.size.bytes.percentile.* + type: object + description: Request size percentiles + - name: request.size.bytes.sum + type: long + format: bytes + unit: byte + metric_type: counter + description: Request size cumulative sum + - name: request.size.bytes.count + type: long + unit: byte + metric_type: counter + description: Request count for size + - name: response.size.bytes.percentile.* + type: object + description: Response size percentiles + - name: response.size.bytes.sum + type: long + format: bytes + unit: byte + metric_type: counter + description: Response size cumulative sum + - name: response.size.bytes.count + type: long + metric_type: counter + description: Response count + - name: request.count + type: long + metric_type: counter + description: Request count for response + - name: client.request.count + type: long + metric_type: counter + description: Number of requests as client + - name: request + type: group + fields: + - name: count + type: long + metric_type: counter + description: Number of requests + - name: latency.sum + type: long + metric_type: counter + description: Requests latency, sum of latencies in microseconds + - name: latency.count + type: long + metric_type: counter + description: Request latency, number of requests + - name: latency.bucket.* + type: object + description: Request latency histogram buckets + - name: duration.us.sum + type: long + metric_type: counter + description: Request duration, sum in microseconds + - name: duration.us.count + type: long + metric_type: counter + description: Request duration, number of operations + - name: duration.us.bucket.* + type: object + description: Request duration, histogram buckets + - name: current.count + type: long + metric_type: counter + description: Inflight requests + - name: longrunning.count + type: long + metric_type: counter + description: Number of requests active long running requests + - name: etcd.object.count + type: long + metric_type: gauge + description: Number of kubernetes objects at etcd + - name: audit.event.count + type: long + metric_type: counter + description: Number of audit events + - name: audit.rejected.count + type: long + metric_type: counter + description: Number of audit rejected events diff --git a/test/packages/kubernetes/data_stream/apiserver/manifest.yml b/test/packages/kubernetes/data_stream/apiserver/manifest.yml new file mode 100644 index 0000000000..cb9eac9cbe --- /dev/null +++ b/test/packages/kubernetes/data_stream/apiserver/manifest.yml @@ -0,0 +1,44 @@ +title: Kubernetes API Server metrics +type: metrics +streams: + - input: kubernetes/metrics + vars: + - name: bearer_token_file + type: text + title: Bearer Token File + multi: false + required: true + show_user: true + default: /var/run/secrets/kubernetes.io/serviceaccount/token + - name: hosts + type: text + title: Hosts + multi: true + required: true + show_user: true + default: + - https://${env.KUBERNETES_SERVICE_HOST}:${env.KUBERNETES_SERVICE_PORT} + - name: leaderelection + type: bool + title: Leader Election + multi: false + required: true + show_user: true + default: true + - name: period + type: text + title: Period + multi: false + required: true + show_user: true + default: 30s + - name: ssl.certificate_authorities + type: text + title: SSL Certificate Authorities + multi: true + required: true + show_user: true + default: + - /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + title: Kubernetes API Server metrics + description: Collect Kubernetes API Server metrics diff --git a/test/packages/kubernetes/data_stream/apiserver/sample_event.json b/test/packages/kubernetes/data_stream/apiserver/sample_event.json new file mode 100644 index 0000000000..fda7d3222a --- /dev/null +++ b/test/packages/kubernetes/data_stream/apiserver/sample_event.json @@ -0,0 +1,62 @@ +{ + "@timestamp": "2020-06-25T12:30:34.616Z", + "metricset": { + "name": "apiserver", + "period": 30000 + }, + "service": { + "address": "10.96.0.1:443", + "type": "kubernetes" + }, + "event": { + "dataset": "kubernetes.apiserver", + "module": "kubernetes", + "duration": 114780772 + }, + "kubernetes": { + "apiserver": { + "request": { + "client": "metrics-server/v0.0.0 (linux/amd64) kubernetes/$Format", + "version": "v1", + "count": 3, + "scope": "cluster", + "content_type": "application/vnd.kubernetes.protobuf", + "code": "200", + "verb": "LIST", + "component": "apiserver", + "resource": "nodes" + } + } + }, + "ecs": { + "version": "1.5.0" + }, + "agent": { + "version": "8.0.0", + "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", + "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "type": "metricbeat" + }, + "host": { + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false, + "ip": [ + "172.17.0.11" + ], + "mac": [ + "02:42:ac:11:00:0b" + ], + "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "architecture": "x86_64", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "os": { + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81", + "codename": "Core", + "platform": "centos", + "version": "7 (Core)" + } + } +} \ No newline at end of file diff --git a/test/packages/kubernetes/data_stream/container/_dev/deploy/k8s/.empty b/test/packages/kubernetes/data_stream/container/_dev/deploy/k8s/.empty new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/packages/kubernetes/data_stream/container/_dev/test/system/test-default-config.yml b/test/packages/kubernetes/data_stream/container/_dev/test/system/test-default-config.yml new file mode 100644 index 0000000000..6c69562d49 --- /dev/null +++ b/test/packages/kubernetes/data_stream/container/_dev/test/system/test-default-config.yml @@ -0,0 +1,7 @@ +service: kubernetes +data_stream: + vars: + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + period: 5s + hosts: + - https://{{Hostname}}:10250 diff --git a/test/packages/kubernetes/data_stream/container/agent/stream/stream.yml.hbs b/test/packages/kubernetes/data_stream/container/agent/stream/stream.yml.hbs new file mode 100644 index 0000000000..bdfe1999ea --- /dev/null +++ b/test/packages/kubernetes/data_stream/container/agent/stream/stream.yml.hbs @@ -0,0 +1,11 @@ +metricsets: ["container"] +add_metadata: {{add_metadata}} +hosts: +{{#each hosts}} + - {{this}} +{{/each}} +period: {{period}} +{{#if bearer_token_file}} +bearer_token_file: {{bearer_token_file}} +ssl.verification_mode: {{ssl.verification_mode}} +{{/if}} diff --git a/test/packages/kubernetes/data_stream/container/fields/agent.yml b/test/packages/kubernetes/data_stream/container/fields/agent.yml new file mode 100644 index 0000000000..d16c882552 --- /dev/null +++ b/test/packages/kubernetes/data_stream/container/fields/agent.yml @@ -0,0 +1,199 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + dimension: true + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + diff --git a/test/packages/kubernetes/data_stream/container/fields/base-fields.yml b/test/packages/kubernetes/data_stream/container/fields/base-fields.yml new file mode 100644 index 0000000000..98c09161d6 --- /dev/null +++ b/test/packages/kubernetes/data_stream/container/fields/base-fields.yml @@ -0,0 +1,98 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. +- name: kubernetes + type: group + fields: + - name: pod.name + dimension: true + type: keyword + description: > + Kubernetes pod name + + - name: pod.uid + dimension: true + type: keyword + description: > + Kubernetes pod UID + + - name: pod.ip + type: ip + description: > + Kubernetes pod IP + + - name: namespace + type: keyword + description: > + Kubernetes namespace + + - name: node.name + type: keyword + description: > + Kubernetes node name + + - name: node.hostname + type: keyword + description: > + Kubernetes hostname as reported by the node’s kernel + + - name: labels.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes labels map + + - name: annotations.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes annotations map + + - name: selectors.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes Service selectors map + + - name: replicaset.name + type: keyword + description: > + Kubernetes replicaset name + + - name: deployment.name + type: keyword + description: > + Kubernetes deployment name + + - name: daemonset.name + type: keyword + description: > + Kubernetes daemonset name + + - name: statefulset.name + type: keyword + description: > + Kubernetes statefulset name + + - name: container.name + dimension: true + type: keyword + description: > + Kubernetes container name + + - name: container.image + type: keyword + description: >- + Kubernetes container image diff --git a/test/packages/kubernetes/data_stream/container/fields/ecs.yml b/test/packages/kubernetes/data_stream/container/fields/ecs.yml new file mode 100644 index 0000000000..cd4e3a89b1 --- /dev/null +++ b/test/packages/kubernetes/data_stream/container/fields/ecs.yml @@ -0,0 +1,10 @@ +- external: ecs + name: ecs.version +- external: ecs + name: service.address +- external: ecs + name: service.type +- external: ecs + name: orchestrator.cluster.name +- external: ecs + name: orchestrator.cluster.url diff --git a/test/packages/kubernetes/data_stream/container/fields/fields.yml b/test/packages/kubernetes/data_stream/container/fields/fields.yml new file mode 100644 index 0000000000..f09af30a34 --- /dev/null +++ b/test/packages/kubernetes/data_stream/container/fields/fields.yml @@ -0,0 +1,199 @@ +- name: kubernetes.container + type: group + fields: + - name: start_time + type: date + description: | + Start time + - name: cpu + type: group + fields: + - name: usage + type: group + fields: + - name: core + type: group + fields: + - name: ns + type: long + metric_type: gauge + description: | + Container CPU Core usage nanoseconds + - name: nanocores + type: long + metric_type: gauge + description: | + CPU used nanocores + - name: node.pct + type: scaled_float + format: percent + unit: percent + metric_type: gauge + description: | + CPU usage as a percentage of the total node allocatable CPU + - name: limit.pct + type: scaled_float + format: percent + unit: percent + metric_type: gauge + description: | + CPU usage as a percentage of the defined limit for the container (or total node allocatable CPU if unlimited) + - name: logs + type: group + fields: + - name: available + type: group + fields: + - name: bytes + type: long + format: bytes + unit: byte + metric_type: gauge + description: | + Logs available capacity in bytes + - name: capacity + type: group + fields: + - name: bytes + type: long + format: bytes + unit: byte + metric_type: gauge + description: | + Logs total capacity in bytes + - name: used + type: group + fields: + - name: bytes + type: long + format: bytes + unit: byte + metric_type: gauge + description: | + Logs used capacity in bytes + - name: inodes + type: group + fields: + - name: count + type: long + metric_type: gauge + description: | + Total available inodes + - name: free + type: long + metric_type: gauge + description: | + Total free inodes + - name: used + type: long + metric_type: gauge + description: | + Total used inodes + - name: memory + type: group + fields: + - name: available + type: group + fields: + - name: bytes + type: long + format: bytes + unit: byte + metric_type: gauge + description: | + Total available memory + - name: usage + type: group + fields: + - name: bytes + type: long + format: bytes + unit: byte + metric_type: gauge + description: | + Total memory usage + - name: node.pct + type: scaled_float + format: percent + unit: percent + metric_type: gauge + description: | + Memory usage as a percentage of the total node allocatable memory + - name: limit.pct + type: scaled_float + format: percent + unit: percent + metric_type: gauge + description: | + Memory usage as a percentage of the defined limit for the container (or total node allocatable memory if unlimited) + - name: rss + type: group + fields: + - name: bytes + type: long + format: bytes + unit: byte + metric_type: gauge + description: | + RSS memory usage + - name: workingset + type: group + fields: + - name: bytes + type: long + format: bytes + unit: byte + metric_type: gauge + description: | + Working set memory usage + - name: pagefaults + type: long + metric_type: counter + description: | + Number of page faults + - name: majorpagefaults + type: long + metric_type: counter + description: | + Number of major page faults + - name: rootfs + type: group + fields: + - name: capacity + type: group + fields: + - name: bytes + type: long + format: bytes + unit: byte + metric_type: gauge + description: | + Root filesystem total capacity in bytes + - name: available + type: group + fields: + - name: bytes + type: long + format: bytes + unit: byte + metric_type: gauge + description: | + Root filesystem total available in bytes + - name: used + type: group + fields: + - name: bytes + type: long + format: bytes + unit: byte + metric_type: gauge + description: | + Root filesystem total used in bytes + - name: inodes + type: group + fields: + - name: used + type: long + metric_type: gauge + description: | + Used inodes diff --git a/test/packages/kubernetes/data_stream/container/manifest.yml b/test/packages/kubernetes/data_stream/container/manifest.yml new file mode 100644 index 0000000000..66377c9994 --- /dev/null +++ b/test/packages/kubernetes/data_stream/container/manifest.yml @@ -0,0 +1,43 @@ +title: Kubernetes Container metrics +type: metrics +streams: + - input: kubernetes/metrics + vars: + - name: add_metadata + type: bool + title: Add Metadata + multi: false + required: true + show_user: true + default: true + - name: bearer_token_file + type: text + title: Bearer Token File + multi: false + required: true + show_user: true + default: /var/run/secrets/kubernetes.io/serviceaccount/token + - name: hosts + type: text + title: Hosts + multi: true + required: true + show_user: true + default: + - https://${env.NODE_NAME}:10250 + - name: period + type: text + title: Period + multi: false + required: true + show_user: true + default: 10s + - name: ssl.verification_mode + type: text + title: SSL Verification Mode + multi: false + required: true + show_user: true + default: none + title: Kubernetes Container metrics + description: Collect Kubernetes Container metrics diff --git a/test/packages/kubernetes/data_stream/container/sample_event.json b/test/packages/kubernetes/data_stream/container/sample_event.json new file mode 100644 index 0000000000..2bbe7c0a93 --- /dev/null +++ b/test/packages/kubernetes/data_stream/container/sample_event.json @@ -0,0 +1,150 @@ +{ + "@timestamp": "2020-06-25T12:32:29.748Z", + "kubernetes": { + "namespace": "kube-system", + "node": { + "name": "minikube" + }, + "pod": { + "name": "metricbeat-g9fc6" + }, + "container": { + "rootfs": { + "used": { + "bytes": 61440 + }, + "inodes": { + "used": 17 + }, + "available": { + "bytes": 6724222976 + }, + "capacity": { + "bytes": 17361141760 + } + }, + "logs": { + "used": { + "bytes": 1617920 + }, + "inodes": { + "count": 9768928, + "used": 223910, + "free": 9545018 + }, + "available": { + "bytes": 6724222976 + }, + "capacity": { + "bytes": 17361141760 + } + }, + "start_time": "2020-06-25T07:19:37Z", + "name": "metricbeat", + "cpu": { + "usage": { + "node": { + "pct": 0.00015289625 + }, + "limit": { + "pct": 0.00015289625 + }, + "nanocores": 611585, + "core": { + "ns": 12206519774 + } + } + }, + "memory": { + "pagefaults": 10164, + "majorpagefaults": 528, + "available": { + "bytes": 188600320 + }, + "usage": { + "limit": { + "pct": 0.005608354460473573 + }, + "bytes": 94306304, + "node": { + "pct": 0.005608354460473573 + } + }, + "workingset": { + "bytes": 21114880 + }, + "rss": { + "bytes": 18386944 + } + } + } + }, + "host": { + "containerized": false, + "ip": [ + "192.168.64.10", + "fe80::a883:2fff:fe7f:6b12", + "172.17.0.1", + "fe80::42:d4ff:fe8c:9493", + "fe80::2859:80ff:fe9e:fcd6", + "fe80::d83a:d9ff:fee9:7052", + "fe80::880a:b6ff:fe18:ba76", + "fe80::f447:faff:fe80:e88b", + "fe80::9cc3:ffff:fe95:e48e", + "fe80::6c1c:29ff:fe50:d40c", + "fe80::b4f3:11ff:fe60:14ed", + "fe80::20f2:2aff:fe96:1e7b", + "fe80::5434:baff:fede:5720", + "fe80::a878:91ff:fe29:81f7" + ], + "mac": [ + "aa:83:2f:7f:6b:12", + "02:42:d4:8c:94:93", + "2a:59:80:9e:fc:d6", + "da:3a:d9:e9:70:52", + "8a:0a:b6:18:ba:76", + "f6:47:fa:80:e8:8b", + "9e:c3:ff:95:e4:8e", + "6e:1c:29:50:d4:0c", + "b6:f3:11:60:14:ed", + "22:f2:2a:96:1e:7b", + "56:34:ba:de:57:20", + "aa:78:91:29:81:f7" + ], + "hostname": "minikube", + "architecture": "x86_64", + "os": { + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81", + "codename": "Core", + "platform": "centos", + "version": "7 (Core)" + }, + "name": "minikube", + "id": "b0e83d397c054b8a99a431072fe4617b" + }, + "agent": { + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "b964a246-96c0-456a-a5c2-8c8b1040ecaf", + "id": "f7ec69f9-4997-4e76-b6c7-0c75206b727a", + "name": "minikube" + }, + "metricset": { + "period": 10000, + "name": "container" + }, + "service": { + "address": "minikube:10250", + "type": "kubernetes" + }, + "event": { + "dataset": "kubernetes.container", + "module": "kubernetes", + "duration": 11091346 + }, + "ecs": { + "version": "1.5.0" + } +} \ No newline at end of file diff --git a/test/packages/kubernetes/data_stream/container_logs/agent/stream/stream.yml.hbs b/test/packages/kubernetes/data_stream/container_logs/agent/stream/stream.yml.hbs new file mode 100644 index 0000000000..9432fd0a1a --- /dev/null +++ b/test/packages/kubernetes/data_stream/container_logs/agent/stream/stream.yml.hbs @@ -0,0 +1,7 @@ +paths: +{{#each paths}} + - {{this}} +{{/each}} +prospector.scanner.symlinks: {{ symlinks }} +parsers: + - container: ~ diff --git a/test/packages/kubernetes/data_stream/container_logs/fields/agent.yml b/test/packages/kubernetes/data_stream/container_logs/fields/agent.yml new file mode 100644 index 0000000000..5959b701dc --- /dev/null +++ b/test/packages/kubernetes/data_stream/container_logs/fields/agent.yml @@ -0,0 +1,200 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + dimension: true + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + dimension: true + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + diff --git a/test/packages/kubernetes/data_stream/container_logs/fields/base-fields.yml b/test/packages/kubernetes/data_stream/container_logs/fields/base-fields.yml new file mode 100644 index 0000000000..21f9fc16fb --- /dev/null +++ b/test/packages/kubernetes/data_stream/container_logs/fields/base-fields.yml @@ -0,0 +1,104 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. +- name: log.offset + type: long + description: Offset of the entry in the log file. +- name: log.file.path + type: keyword + description: Path to the log file. +- name: kubernetes + type: group + fields: + - name: pod.name + dimension: true + type: keyword + description: > + Kubernetes pod name + + - name: pod.uid + dimension: true + type: keyword + description: > + Kubernetes pod UID + + - name: pod.ip + type: ip + description: > + Kubernetes pod IP + + - name: namespace + type: keyword + description: > + Kubernetes namespace + + - name: node.name + type: keyword + description: > + Kubernetes node name + + - name: node.hostname + type: keyword + description: > + Kubernetes hostname as reported by the node’s kernel + + - name: labels.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes labels map + + - name: annotations.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes annotations map + + - name: selectors.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes Service selectors map + + - name: replicaset.name + type: keyword + description: > + Kubernetes replicaset name + + - name: deployment.name + type: keyword + description: > + Kubernetes deployment name + + - name: daemonset.name + type: keyword + description: > + Kubernetes daemonset name + + - name: statefulset.name + type: keyword + description: > + Kubernetes statefulset name + + - name: container.name + dimension: true + type: keyword + description: > + Kubernetes container name + + - name: container.image + type: keyword + description: >- + Kubernetes container image diff --git a/test/packages/kubernetes/data_stream/container_logs/fields/ecs.yml b/test/packages/kubernetes/data_stream/container_logs/fields/ecs.yml new file mode 100644 index 0000000000..f6818be260 --- /dev/null +++ b/test/packages/kubernetes/data_stream/container_logs/fields/ecs.yml @@ -0,0 +1,18 @@ +- external: ecs + name: ecs.version +- external: ecs + name: error.message +- external: ecs + name: event.ingested +- external: ecs + name: agent.name +- external: ecs + name: agent.type +- external: ecs + name: agent.id +- external: ecs + name: agent.ephemeral_id +- external: ecs + name: agent.version +- external: ecs + name: message diff --git a/test/packages/kubernetes/data_stream/container_logs/manifest.yml b/test/packages/kubernetes/data_stream/container_logs/manifest.yml new file mode 100644 index 0000000000..19bf942d94 --- /dev/null +++ b/test/packages/kubernetes/data_stream/container_logs/manifest.yml @@ -0,0 +1,21 @@ +title: "Kubernetes container logs" +type: logs +streams: + - input: filestream + title: Collect Kubernetes container logs + description: Collect Kubernetes container logs + vars: + - name: paths + type: text + required: true + title: Kubernetes container log path + multi: true + default: + - /var/log/containers/*${kubernetes.container.id}.log + - name: symlinks + type: bool + title: Use Symlinks + multi: false + required: true + show_user: true + default: true diff --git a/test/packages/kubernetes/data_stream/controllermanager/_dev/deploy/k8s/.empty b/test/packages/kubernetes/data_stream/controllermanager/_dev/deploy/k8s/.empty new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/packages/kubernetes/data_stream/controllermanager/_dev/test/system/test-default-config.yml b/test/packages/kubernetes/data_stream/controllermanager/_dev/test/system/test-default-config.yml new file mode 100644 index 0000000000..60403494f7 --- /dev/null +++ b/test/packages/kubernetes/data_stream/controllermanager/_dev/test/system/test-default-config.yml @@ -0,0 +1,8 @@ +service: kubernetes +data_stream: + vars: + period: 5s + hosts: + - https://0.0.0.0:10257 + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + ssl.verification_mode: "none" diff --git a/test/packages/kubernetes/data_stream/controllermanager/agent/stream/stream.yml.hbs b/test/packages/kubernetes/data_stream/controllermanager/agent/stream/stream.yml.hbs new file mode 100644 index 0000000000..227746923f --- /dev/null +++ b/test/packages/kubernetes/data_stream/controllermanager/agent/stream/stream.yml.hbs @@ -0,0 +1,14 @@ +metricsets: ["controllermanager"] + +hosts: +{{#each hosts}} + - {{this}} +{{/each}} +period: {{period}} + +{{#if bearer_token_file}} +bearer_token_file: {{bearer_token_file}} +ssl.verification_mode: {{ssl.verification_mode}} +{{/if}} + +condition: ${kubernetes.labels.{{~controller_manager_label_key~}} } == '{{controller_manager_label_value}}' \ No newline at end of file diff --git a/test/packages/kubernetes/data_stream/controllermanager/fields/agent.yml b/test/packages/kubernetes/data_stream/controllermanager/fields/agent.yml new file mode 100644 index 0000000000..da4e652c53 --- /dev/null +++ b/test/packages/kubernetes/data_stream/controllermanager/fields/agent.yml @@ -0,0 +1,198 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + diff --git a/test/packages/kubernetes/data_stream/controllermanager/fields/base-fields.yml b/test/packages/kubernetes/data_stream/controllermanager/fields/base-fields.yml new file mode 100644 index 0000000000..d43ffd7964 --- /dev/null +++ b/test/packages/kubernetes/data_stream/controllermanager/fields/base-fields.yml @@ -0,0 +1,90 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. +- name: kubernetes + type: group + fields: + - name: pod.name + type: keyword + description: > + Kubernetes pod name + + - name: pod.uid + type: keyword + description: > + Kubernetes pod UID + + - name: pod.ip + type: ip + description: > + Kubernetes pod IP + + - name: namespace + type: keyword + description: > + Kubernetes namespace + + - name: node.name + type: keyword + description: > + Kubernetes node name + + - name: node.hostname + type: keyword + description: > + Kubernetes hostname as reported by the node’s kernel + + - name: labels.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes labels map + + - name: annotations.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes annotations map + + - name: selectors.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes Service selectors map + + - name: replicaset.name + type: keyword + description: > + Kubernetes replicaset name + + - name: deployment.name + type: keyword + description: > + Kubernetes deployment name + + - name: statefulset.name + type: keyword + description: > + Kubernetes statefulset name + + - name: container.name + type: keyword + description: > + Kubernetes container name + + - name: container.image + type: keyword + description: >- + Kubernetes container image diff --git a/test/packages/kubernetes/data_stream/controllermanager/fields/ecs.yml b/test/packages/kubernetes/data_stream/controllermanager/fields/ecs.yml new file mode 100644 index 0000000000..cd4e3a89b1 --- /dev/null +++ b/test/packages/kubernetes/data_stream/controllermanager/fields/ecs.yml @@ -0,0 +1,10 @@ +- external: ecs + name: ecs.version +- external: ecs + name: service.address +- external: ecs + name: service.type +- external: ecs + name: orchestrator.cluster.name +- external: ecs + name: orchestrator.cluster.url diff --git a/test/packages/kubernetes/data_stream/controllermanager/fields/fields.yml b/test/packages/kubernetes/data_stream/controllermanager/fields/fields.yml new file mode 100644 index 0000000000..1ef76f93e8 --- /dev/null +++ b/test/packages/kubernetes/data_stream/controllermanager/fields/fields.yml @@ -0,0 +1,159 @@ +- name: kubernetes.controllermanager + type: group + fields: + - name: handler + dimension: true + type: keyword + description: | + Request handler + - name: code + dimension: true + type: keyword + description: | + HTTP code + - name: method + dimension: true + type: keyword + description: | + HTTP method + - name: host + dimension: true + type: keyword + description: | + Request host + - name: name + dimension: true + type: keyword + description: | + Name for the resource + - name: zone + dimension: true + type: keyword + description: | + Infrastructure zone + - name: process + type: group + fields: + - name: cpu.sec + type: double + metric_type: counter + description: CPU seconds + - name: memory.resident.bytes + type: long + format: bytes + unit: byte + metric_type: gauge + description: Bytes in resident memory + - name: memory.virtual.bytes + type: long + format: bytes + unit: byte + metric_type: gauge + description: Bytes in virtual memory + - name: fds.open.count + type: long + metric_type: gauge + description: Number of open file descriptors + - name: started.sec + type: double + metric_type: gauge + description: Seconds since the process started + - name: http + type: group + fields: + - name: request.duration.us.percentile.* + type: object + description: Request duration microseconds percentiles + - name: request.duration.us.sum + type: double + unit: micros + metric_type: counter + description: Request duration microseconds cumulative sum + - name: request.duration.us.count + type: long + unit: micros + metric_type: counter + description: Request count for duration + - name: request.size.bytes.percentile.* + type: object + description: Request size percentiles + - name: request.size.bytes.sum + type: long + format: bytes + unit: byte + metric_type: counter + description: Request size cumulative sum + - name: request.size.bytes.count + type: long + unit: byte + metric_type: counter + description: Request count for size + - name: response.size.bytes.percentile.* + type: object + description: Response size percentiles + - name: response.size.bytes.sum + type: long + format: bytes + unit: byte + metric_type: counter + description: Response size cumulative sum + - name: response.size.bytes.count + type: long + unit: byte + metric_type: counter + description: Response count + - name: request.count + type: long + metric_type: counter + description: Request count for response + - name: client.request.count + type: long + metric_type: counter + description: | + Number of requests as client + - name: workqueue + type: group + fields: + - name: longestrunning.sec + type: double + metric_type: gauge + description: Longest running processors + - name: unfinished.sec + type: double + metric_type: gauge + description: Unfinished processors + - name: adds.count + type: long + metric_type: counter + description: Workqueue add count + - name: depth.count + type: long + metric_type: gauge + description: Workqueue depth count + - name: retries.count + type: long + metric_type: counter + description: Workqueue number of retries + - name: node.collector + type: group + fields: + - name: eviction.count + type: long + metric_type: counter + description: Number of node evictions + - name: unhealthy.count + type: long + metric_type: gauge + description: Number of unhealthy nodes + - name: count + type: long + metric_type: gauge + description: Number of nodes + - name: health.pct + type: long + metric_type: gauge + description: Percentage of healthy nodes + - name: leader.is_master + type: boolean + description: | + Whether the node is master diff --git a/test/packages/kubernetes/data_stream/controllermanager/manifest.yml b/test/packages/kubernetes/data_stream/controllermanager/manifest.yml new file mode 100644 index 0000000000..12e0f56da5 --- /dev/null +++ b/test/packages/kubernetes/data_stream/controllermanager/manifest.yml @@ -0,0 +1,51 @@ +title: Kubernetes Controller Manager metrics +type: metrics +streams: + - input: kubernetes/metrics + enabled: false + vars: + - name: bearer_token_file + type: text + title: Bearer Token File + multi: false + required: true + show_user: true + default: /var/run/secrets/kubernetes.io/serviceaccount/token + - name: hosts + type: text + title: Hosts + multi: true + required: true + show_user: true + default: + - https://0.0.0.0:10257 + - name: period + type: text + title: Period + multi: false + required: true + show_user: true + default: 10s + - name: ssl.verification_mode + type: text + title: SSL Verification Mode + multi: false + required: true + show_user: true + default: none + - name: controller_manager_label_key + type: text + title: Kubernetes Controller Manager Label key + multi: false + required: true + show_user: false + default: component + - name: controller_manager_label_value + type: text + title: Kubernetes Controller Manager Label value + multi: false + required: true + show_user: false + default: kube-controller-manager + title: Kubernetes Controller Manager metrics + description: Collect Kubernetes Controller Manager metrics diff --git a/test/packages/kubernetes/data_stream/controllermanager/sample_event.json b/test/packages/kubernetes/data_stream/controllermanager/sample_event.json new file mode 100644 index 0000000000..7aa39a4dd4 --- /dev/null +++ b/test/packages/kubernetes/data_stream/controllermanager/sample_event.json @@ -0,0 +1,93 @@ +{ + "@timestamp": "2020-06-25T12:33:29.643Z", + "kubernetes": { + "controllermanager": { + "workqueue": { + "unfinished": { + "sec": 0 + }, + "adds": { + "count": 0 + }, + "depth": { + "count": 0 + }, + "longestrunning": { + "sec": 0 + }, + "retries": { + "count": 0 + } + }, + "name": "certificate" + } + }, + "event": { + "dataset": "kubernetes.controllermanager", + "module": "kubernetes", + "duration": 8893806 + }, + "ecs": { + "version": "1.5.0" + }, + "host": { + "ip": [ + "192.168.64.10", + "fe80::a883:2fff:fe7f:6b12", + "172.17.0.1", + "fe80::42:d4ff:fe8c:9493", + "fe80::2859:80ff:fe9e:fcd6", + "fe80::d83a:d9ff:fee9:7052", + "fe80::880a:b6ff:fe18:ba76", + "fe80::f447:faff:fe80:e88b", + "fe80::9cc3:ffff:fe95:e48e", + "fe80::6c1c:29ff:fe50:d40c", + "fe80::b4f3:11ff:fe60:14ed", + "fe80::20f2:2aff:fe96:1e7b", + "fe80::5434:baff:fede:5720", + "fe80::a878:91ff:fe29:81f7" + ], + "mac": [ + "aa:83:2f:7f:6b:12", + "02:42:d4:8c:94:93", + "2a:59:80:9e:fc:d6", + "da:3a:d9:e9:70:52", + "8a:0a:b6:18:ba:76", + "f6:47:fa:80:e8:8b", + "9e:c3:ff:95:e4:8e", + "6e:1c:29:50:d4:0c", + "b6:f3:11:60:14:ed", + "22:f2:2a:96:1e:7b", + "56:34:ba:de:57:20", + "aa:78:91:29:81:f7" + ], + "hostname": "minikube", + "architecture": "x86_64", + "os": { + "codename": "Core", + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81" + }, + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false, + "name": "minikube" + }, + "agent": { + "version": "8.0.0", + "ephemeral_id": "b964a246-96c0-456a-a5c2-8c8b1040ecaf", + "id": "f7ec69f9-4997-4e76-b6c7-0c75206b727a", + "name": "minikube", + "type": "metricbeat" + }, + "metricset": { + "period": 10000, + "name": "controllermanager" + }, + "service": { + "address": "localhost:10252", + "type": "kubernetes" + } +} \ No newline at end of file diff --git a/test/packages/kubernetes/data_stream/event/agent/stream/stream.yml.hbs b/test/packages/kubernetes/data_stream/event/agent/stream/stream.yml.hbs new file mode 100644 index 0000000000..fe1ecc78ce --- /dev/null +++ b/test/packages/kubernetes/data_stream/event/agent/stream/stream.yml.hbs @@ -0,0 +1,6 @@ +metricsets: ["event"] +period: {{period}} +add_metadata: {{add_metadata}} +{{#if leaderelection}} +condition: ${kubernetes_leaderelection.leader} == true +{{/if}} \ No newline at end of file diff --git a/test/packages/kubernetes/data_stream/event/fields/agent.yml b/test/packages/kubernetes/data_stream/event/fields/agent.yml new file mode 100644 index 0000000000..da4e652c53 --- /dev/null +++ b/test/packages/kubernetes/data_stream/event/fields/agent.yml @@ -0,0 +1,198 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + diff --git a/test/packages/kubernetes/data_stream/event/fields/base-fields.yml b/test/packages/kubernetes/data_stream/event/fields/base-fields.yml new file mode 100644 index 0000000000..d43ffd7964 --- /dev/null +++ b/test/packages/kubernetes/data_stream/event/fields/base-fields.yml @@ -0,0 +1,90 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. +- name: kubernetes + type: group + fields: + - name: pod.name + type: keyword + description: > + Kubernetes pod name + + - name: pod.uid + type: keyword + description: > + Kubernetes pod UID + + - name: pod.ip + type: ip + description: > + Kubernetes pod IP + + - name: namespace + type: keyword + description: > + Kubernetes namespace + + - name: node.name + type: keyword + description: > + Kubernetes node name + + - name: node.hostname + type: keyword + description: > + Kubernetes hostname as reported by the node’s kernel + + - name: labels.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes labels map + + - name: annotations.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes annotations map + + - name: selectors.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes Service selectors map + + - name: replicaset.name + type: keyword + description: > + Kubernetes replicaset name + + - name: deployment.name + type: keyword + description: > + Kubernetes deployment name + + - name: statefulset.name + type: keyword + description: > + Kubernetes statefulset name + + - name: container.name + type: keyword + description: > + Kubernetes container name + + - name: container.image + type: keyword + description: >- + Kubernetes container image diff --git a/test/packages/kubernetes/data_stream/event/fields/ecs.yml b/test/packages/kubernetes/data_stream/event/fields/ecs.yml new file mode 100644 index 0000000000..32165e66a4 --- /dev/null +++ b/test/packages/kubernetes/data_stream/event/fields/ecs.yml @@ -0,0 +1,8 @@ +- external: ecs + name: ecs.version +- external: ecs + name: service.type +- external: ecs + name: orchestrator.cluster.name +- external: ecs + name: orchestrator.cluster.url diff --git a/test/packages/kubernetes/data_stream/event/fields/fields.yml b/test/packages/kubernetes/data_stream/event/fields/fields.yml new file mode 100644 index 0000000000..9bcbf00016 --- /dev/null +++ b/test/packages/kubernetes/data_stream/event/fields/fields.yml @@ -0,0 +1,108 @@ +- name: kubernetes.event + type: group + fields: + - name: count + type: long + metric_type: counter + description: | + Count field records the number of times the particular event has occurred + - name: timestamp + type: group + fields: + - name: first_occurrence + type: date + description: | + Timestamp of first occurrence of event + - name: last_occurrence + type: date + description: | + Timestamp of last occurrence of event + - name: message + type: text + description: | + Message recorded for the given event + - name: reason + dimension: true + type: keyword + description: | + Reason recorded for the given event + - name: type + dimension: true + type: keyword + description: | + Type of the given event + - name: source + type: group + fields: + - name: component + dimension: true + type: keyword + description: | + Component from which the event is generated + - name: host + dimension: true + type: keyword + description: | + Node name on which the event is generated + - name: metadata + type: group + fields: + - name: timestamp + type: group + fields: + - name: created + type: date + description: | + Timestamp of creation of the given event + - name: generate_name + dimension: true + type: keyword + description: | + Generate name of the event + - name: name + dimension: true + type: keyword + description: | + Name of the event + - name: namespace + dimension: true + type: keyword + description: | + Namespace in which event was generated + - name: resource_version + dimension: true + type: keyword + description: | + Version of the event resource + - name: uid + type: keyword + description: | + Unique identifier to the event object + - name: self_link + type: keyword + description: | + URL representing the event + - name: involved_object + type: group + fields: + - name: api_version + type: keyword + description: | + API version of the object + - name: kind + type: keyword + description: | + API kind of the object + - name: name + type: keyword + description: | + name of the object + - name: resource_version + type: keyword + description: | + resource version of the object + - name: uid + dimension: true + type: keyword + description: | + uid version of the object diff --git a/test/packages/kubernetes/data_stream/event/manifest.yml b/test/packages/kubernetes/data_stream/event/manifest.yml new file mode 100644 index 0000000000..cecc6ef73c --- /dev/null +++ b/test/packages/kubernetes/data_stream/event/manifest.yml @@ -0,0 +1,28 @@ +title: Kubernetes Event metrics +type: metrics +streams: + - input: kubernetes/metrics + vars: + - name: period + type: text + title: Period + multi: false + required: true + show_user: true + default: 10s + - name: add_metadata + type: bool + title: Add Metadata + multi: false + required: true + show_user: true + default: true + - name: leaderelection + type: bool + title: Leader Election + multi: false + required: true + show_user: true + default: true + title: Kubernetes Event metrics + description: Collect Kubernetes Event metrics diff --git a/test/packages/kubernetes/data_stream/event/sample_event.json b/test/packages/kubernetes/data_stream/event/sample_event.json new file mode 100644 index 0000000000..01c0c93fcf --- /dev/null +++ b/test/packages/kubernetes/data_stream/event/sample_event.json @@ -0,0 +1,78 @@ +{ + "@timestamp": "2020-06-25T12:30:27.575Z", + "metricset": { + "name": "event" + }, + "agent": { + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", + "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc" + }, + "ecs": { + "version": "1.5.0" + }, + "event": { + "dataset": "kubernetes.event", + "module": "kubernetes" + }, + "service": { + "type": "kubernetes" + }, + "kubernetes": { + "event": { + "metadata": { + "uid": "604e39e0-862f-4615-9cec-8cb62299dea3", + "resource_version": "485630", + "timestamp": { + "created": "2020-06-25T07:20:25.000Z" + }, + "name": "monitor.161bb862545e3099", + "namespace": "beats", + "self_link": "/api/v1/namespaces/beats/events/monitor.161bb862545e3099", + "generate_name": "" + }, + "timestamp": { + "first_occurrence": "2020-06-25T07:20:25.000Z", + "last_occurrence": "2020-06-25T12:30:27.000Z" + }, + "message": "Failed to find referenced backend beats/monitor: Elasticsearch.elasticsearch.k8s.elastic.co \"monitor\" not found", + "reason": "AssociationError", + "type": "Warning", + "count": 1861, + "source": { + "host": "", + "component": "kibana-association-controller" + }, + "involved_object": { + "api_version": "kibana.k8s.elastic.co/v1", + "resource_version": "101842", + "name": "monitor", + "kind": "Kibana", + "uid": "45a19de5-5eef-4090-a2d3-dbceb0a28af8" + } + } + }, + "host": { + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false, + "ip": [ + "172.17.0.11" + ], + "mac": [ + "02:42:ac:11:00:0b" + ], + "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "architecture": "x86_64", + "os": { + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81", + "codename": "Core" + } + } +} \ No newline at end of file diff --git a/test/packages/kubernetes/data_stream/node/_dev/deploy/k8s/.empty b/test/packages/kubernetes/data_stream/node/_dev/deploy/k8s/.empty new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/packages/kubernetes/data_stream/node/_dev/test/system/test-default-config.yml b/test/packages/kubernetes/data_stream/node/_dev/test/system/test-default-config.yml new file mode 100644 index 0000000000..6c69562d49 --- /dev/null +++ b/test/packages/kubernetes/data_stream/node/_dev/test/system/test-default-config.yml @@ -0,0 +1,7 @@ +service: kubernetes +data_stream: + vars: + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + period: 5s + hosts: + - https://{{Hostname}}:10250 diff --git a/test/packages/kubernetes/data_stream/node/agent/stream/stream.yml.hbs b/test/packages/kubernetes/data_stream/node/agent/stream/stream.yml.hbs new file mode 100644 index 0000000000..c4a3cbda5e --- /dev/null +++ b/test/packages/kubernetes/data_stream/node/agent/stream/stream.yml.hbs @@ -0,0 +1,11 @@ +metricsets: ["node"] +add_metadata: {{add_metadata}} +hosts: +{{#each hosts}} + - {{this}} +{{/each}} +period: {{period}} +{{#if bearer_token_file}} +bearer_token_file: {{bearer_token_file}} +ssl.verification_mode: {{ssl.verification_mode}} +{{/if}} diff --git a/test/packages/kubernetes/data_stream/node/fields/agent.yml b/test/packages/kubernetes/data_stream/node/fields/agent.yml new file mode 100644 index 0000000000..da4e652c53 --- /dev/null +++ b/test/packages/kubernetes/data_stream/node/fields/agent.yml @@ -0,0 +1,198 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + diff --git a/test/packages/kubernetes/data_stream/node/fields/base-fields.yml b/test/packages/kubernetes/data_stream/node/fields/base-fields.yml new file mode 100644 index 0000000000..0e3072d1ef --- /dev/null +++ b/test/packages/kubernetes/data_stream/node/fields/base-fields.yml @@ -0,0 +1,91 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. +- name: kubernetes + type: group + fields: + - name: pod.name + type: keyword + description: > + Kubernetes pod name + + - name: pod.uid + type: keyword + description: > + Kubernetes pod UID + + - name: pod.ip + type: ip + description: > + Kubernetes pod IP + + - name: namespace + type: keyword + description: > + Kubernetes namespace + + - name: node.name + dimension: true + type: keyword + description: > + Kubernetes node name + + - name: node.hostname + type: keyword + description: > + Kubernetes hostname as reported by the node’s kernel + + - name: labels.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes labels map + + - name: annotations.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes annotations map + + - name: selectors.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes Service selectors map + + - name: replicaset.name + type: keyword + description: > + Kubernetes replicaset name + + - name: deployment.name + type: keyword + description: > + Kubernetes deployment name + + - name: statefulset.name + type: keyword + description: > + Kubernetes statefulset name + + - name: container.name + type: keyword + description: > + Kubernetes container name + + - name: container.image + type: keyword + description: >- + Kubernetes container image diff --git a/test/packages/kubernetes/data_stream/node/fields/ecs.yml b/test/packages/kubernetes/data_stream/node/fields/ecs.yml new file mode 100644 index 0000000000..cd4e3a89b1 --- /dev/null +++ b/test/packages/kubernetes/data_stream/node/fields/ecs.yml @@ -0,0 +1,10 @@ +- external: ecs + name: ecs.version +- external: ecs + name: service.address +- external: ecs + name: service.type +- external: ecs + name: orchestrator.cluster.name +- external: ecs + name: orchestrator.cluster.url diff --git a/test/packages/kubernetes/data_stream/node/fields/fields.yml b/test/packages/kubernetes/data_stream/node/fields/fields.yml new file mode 100644 index 0000000000..dc46f35f2a --- /dev/null +++ b/test/packages/kubernetes/data_stream/node/fields/fields.yml @@ -0,0 +1,198 @@ +- name: kubernetes.node + type: group + fields: + - name: start_time + type: date + description: | + Start time + - name: cpu + type: group + fields: + - name: usage + type: group + fields: + - name: core + type: group + fields: + - name: ns + type: long + metric_type: gauge + description: | + Node CPU Core usage nanoseconds + - name: nanocores + type: long + metric_type: gauge + description: | + CPU used nanocores + - name: memory + type: group + fields: + - name: available + type: group + fields: + - name: bytes + type: long + format: bytes + unit: byte + metric_type: gauge + description: | + Total available memory + - name: usage + type: group + fields: + - name: bytes + type: long + format: bytes + unit: byte + metric_type: gauge + description: | + Total memory usage + - name: rss + type: group + fields: + - name: bytes + type: long + format: bytes + unit: byte + metric_type: gauge + description: | + RSS memory usage + - name: workingset + type: group + fields: + - name: bytes + type: long + format: bytes + unit: byte + metric_type: gauge + description: | + Working set memory usage + - name: pagefaults + type: long + metric_type: counter + description: | + Number of page faults + - name: majorpagefaults + type: long + metric_type: counter + description: | + Number of major page faults + - name: network + type: group + fields: + - name: rx + type: group + fields: + - name: bytes + type: long + format: bytes + unit: byte + metric_type: counter + description: | + Received bytes + - name: errors + type: long + description: | + Rx errors + - name: tx + type: group + fields: + - name: bytes + type: long + format: bytes + unit: byte + metric_type: counter + description: | + Transmitted bytes + - name: errors + type: long + metric_type: counter + description: | + Tx errors + - name: fs + type: group + fields: + - name: capacity + type: group + fields: + - name: bytes + type: long + format: bytes + unit: byte + metric_type: gauge + description: | + Filesystem total capacity in bytes + - name: available + type: group + fields: + - name: bytes + type: long + format: bytes + unit: byte + metric_type: gauge + description: | + Filesystem total available in bytes + - name: used + type: group + fields: + - name: bytes + type: long + format: bytes + unit: byte + metric_type: gauge + description: | + Filesystem total used in bytes + - name: inodes + type: group + fields: + - name: used + type: long + metric_type: gauge + description: | + Number of used inodes + - name: count + type: long + metric_type: gauge + description: | + Number of inodes + - name: free + type: long + metric_type: gauge + description: | + Number of free inodes + - name: runtime + type: group + fields: + - name: imagefs + type: group + fields: + - name: capacity + type: group + fields: + - name: bytes + type: long + format: bytes + unit: byte + metric_type: gauge + description: | + Image filesystem total capacity in bytes + - name: available + type: group + fields: + - name: bytes + type: long + format: bytes + unit: byte + metric_type: gauge + description: | + Image filesystem total available in bytes + - name: used + type: group + fields: + - name: bytes + type: long + format: bytes + unit: byte + metric_type: gauge + description: | + Image filesystem total used in bytes diff --git a/test/packages/kubernetes/data_stream/node/manifest.yml b/test/packages/kubernetes/data_stream/node/manifest.yml new file mode 100644 index 0000000000..fc763864fe --- /dev/null +++ b/test/packages/kubernetes/data_stream/node/manifest.yml @@ -0,0 +1,43 @@ +title: Kubernetes Node metrics +type: metrics +streams: + - input: kubernetes/metrics + vars: + - name: add_metadata + type: bool + title: Add Metadata + multi: false + required: true + show_user: true + default: true + - name: bearer_token_file + type: text + title: Bearer Token File + multi: false + required: true + show_user: true + default: /var/run/secrets/kubernetes.io/serviceaccount/token + - name: hosts + type: text + title: Hosts + multi: true + required: true + show_user: true + default: + - https://${env.NODE_NAME}:10250 + - name: period + type: text + title: Period + multi: false + required: true + show_user: true + default: 10s + - name: ssl.verification_mode + type: text + title: SSL Verification Mode + multi: false + required: true + show_user: true + default: none + title: Kubernetes Node metrics + description: Collect Kubernetes Node metrics diff --git a/test/packages/kubernetes/data_stream/node/sample_event.json b/test/packages/kubernetes/data_stream/node/sample_event.json new file mode 100644 index 0000000000..3128c86a50 --- /dev/null +++ b/test/packages/kubernetes/data_stream/node/sample_event.json @@ -0,0 +1,148 @@ +{ + "@timestamp": "2020-06-25T12:34:39.723Z", + "event": { + "dataset": "kubernetes.node", + "module": "kubernetes", + "duration": 13042307 + }, + "service": { + "type": "kubernetes", + "address": "minikube:10250" + }, + "host": { + "containerized": false, + "ip": [ + "192.168.64.10", + "fe80::a883:2fff:fe7f:6b12", + "172.17.0.1", + "fe80::42:d4ff:fe8c:9493", + "fe80::2859:80ff:fe9e:fcd6", + "fe80::d83a:d9ff:fee9:7052", + "fe80::880a:b6ff:fe18:ba76", + "fe80::f447:faff:fe80:e88b", + "fe80::9cc3:ffff:fe95:e48e", + "fe80::6c1c:29ff:fe50:d40c", + "fe80::b4f3:11ff:fe60:14ed", + "fe80::20f2:2aff:fe96:1e7b", + "fe80::5434:baff:fede:5720", + "fe80::a878:91ff:fe29:81f7" + ], + "mac": [ + "aa:83:2f:7f:6b:12", + "02:42:d4:8c:94:93", + "2a:59:80:9e:fc:d6", + "da:3a:d9:e9:70:52", + "8a:0a:b6:18:ba:76", + "f6:47:fa:80:e8:8b", + "9e:c3:ff:95:e4:8e", + "6e:1c:29:50:d4:0c", + "b6:f3:11:60:14:ed", + "22:f2:2a:96:1e:7b", + "56:34:ba:de:57:20", + "aa:78:91:29:81:f7" + ], + "name": "minikube", + "hostname": "minikube", + "architecture": "x86_64", + "os": { + "codename": "Core", + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81" + }, + "id": "b0e83d397c054b8a99a431072fe4617b" + }, + "metricset": { + "name": "node", + "period": 10000 + }, + "kubernetes": { + "labels": { + "beta_kubernetes_io/os": "linux", + "kubernetes_io/arch": "amd64", + "kubernetes_io/hostname": "minikube", + "kubernetes_io/os": "linux", + "node-role_kubernetes_io/master": "", + "beta_kubernetes_io/arch": "amd64" + }, + "node": { + "memory": { + "available": { + "bytes": 12746428416 + }, + "usage": { + "bytes": 5670916096 + }, + "workingset": { + "bytes": 4068896768 + }, + "rss": { + "bytes": 3252125696 + }, + "pagefaults": 31680, + "majorpagefaults": 0 + }, + "network": { + "rx": { + "bytes": 107077476, + "errors": 0 + }, + "tx": { + "bytes": 67457933, + "errors": 0 + } + }, + "fs": { + "available": { + "bytes": 6655090688 + }, + "capacity": { + "bytes": 17361141760 + }, + "used": { + "bytes": 9689358336 + }, + "inodes": { + "count": 9768928, + "used": 224151, + "free": 9544777 + } + }, + "runtime": { + "imagefs": { + "capacity": { + "bytes": 17361141760 + }, + "used": { + "bytes": 8719928568 + }, + "available": { + "bytes": 6655090688 + } + } + }, + "start_time": "2020-06-25T07:18:38Z", + "name": "minikube", + "cpu": { + "usage": { + "core": { + "ns": 6136184971873 + }, + "nanocores": 455263291 + } + } + } + }, + "agent": { + "name": "minikube", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "b964a246-96c0-456a-a5c2-8c8b1040ecaf", + "id": "f7ec69f9-4997-4e76-b6c7-0c75206b727a" + }, + "ecs": { + "version": "1.5.0" + } +} \ No newline at end of file diff --git a/test/packages/kubernetes/data_stream/pod/_dev/deploy/k8s/.empty b/test/packages/kubernetes/data_stream/pod/_dev/deploy/k8s/.empty new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/packages/kubernetes/data_stream/pod/_dev/test/system/test-default-config.yml b/test/packages/kubernetes/data_stream/pod/_dev/test/system/test-default-config.yml new file mode 100644 index 0000000000..c24e9500a3 --- /dev/null +++ b/test/packages/kubernetes/data_stream/pod/_dev/test/system/test-default-config.yml @@ -0,0 +1,7 @@ +service: kubernetes +data_stream: + vars: + period: 5s + hosts: + - https://{{Hostname}}:10250 + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token diff --git a/test/packages/kubernetes/data_stream/pod/agent/stream/stream.yml.hbs b/test/packages/kubernetes/data_stream/pod/agent/stream/stream.yml.hbs new file mode 100644 index 0000000000..54fd7c22cc --- /dev/null +++ b/test/packages/kubernetes/data_stream/pod/agent/stream/stream.yml.hbs @@ -0,0 +1,11 @@ +metricsets: ["pod"] +add_metadata: {{add_metadata}} +hosts: +{{#each hosts}} + - {{this}} +{{/each}} +period: {{period}} +{{#if bearer_token_file}} +bearer_token_file: {{bearer_token_file}} +ssl.verification_mode: {{ssl.verification_mode}} +{{/if}} diff --git a/test/packages/kubernetes/data_stream/pod/fields/agent.yml b/test/packages/kubernetes/data_stream/pod/fields/agent.yml new file mode 100644 index 0000000000..da4e652c53 --- /dev/null +++ b/test/packages/kubernetes/data_stream/pod/fields/agent.yml @@ -0,0 +1,198 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + diff --git a/test/packages/kubernetes/data_stream/pod/fields/base-fields.yml b/test/packages/kubernetes/data_stream/pod/fields/base-fields.yml new file mode 100644 index 0000000000..5b20fc8d5c --- /dev/null +++ b/test/packages/kubernetes/data_stream/pod/fields/base-fields.yml @@ -0,0 +1,97 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. +- name: kubernetes + type: group + fields: + - name: pod.name + dimension: true + type: keyword + description: > + Kubernetes pod name + + - name: pod.uid + dimension: true + type: keyword + description: > + Kubernetes pod UID + + - name: pod.ip + type: ip + description: > + Kubernetes pod IP + + - name: namespace + type: keyword + description: > + Kubernetes namespace + + - name: node.name + type: keyword + description: > + Kubernetes node name + + - name: node.hostname + type: keyword + description: > + Kubernetes hostname as reported by the node’s kernel + + - name: labels.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes labels map + + - name: annotations.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes annotations map + + - name: selectors.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes Service selectors map + + - name: replicaset.name + type: keyword + description: > + Kubernetes replicaset name + + - name: deployment.name + type: keyword + description: > + Kubernetes deployment name + + - name: daemonset.name + type: keyword + description: > + Kubernetes daemonset name + + - name: statefulset.name + type: keyword + description: > + Kubernetes statefulset name + + - name: container.name + type: keyword + description: > + Kubernetes container name + + - name: container.image + type: keyword + description: >- + Kubernetes container image diff --git a/test/packages/kubernetes/data_stream/pod/fields/ecs.yml b/test/packages/kubernetes/data_stream/pod/fields/ecs.yml new file mode 100644 index 0000000000..cd4e3a89b1 --- /dev/null +++ b/test/packages/kubernetes/data_stream/pod/fields/ecs.yml @@ -0,0 +1,10 @@ +- external: ecs + name: ecs.version +- external: ecs + name: service.address +- external: ecs + name: service.type +- external: ecs + name: orchestrator.cluster.name +- external: ecs + name: orchestrator.cluster.url diff --git a/test/packages/kubernetes/data_stream/pod/fields/fields.yml b/test/packages/kubernetes/data_stream/pod/fields/fields.yml new file mode 100644 index 0000000000..3073996d3b --- /dev/null +++ b/test/packages/kubernetes/data_stream/pod/fields/fields.yml @@ -0,0 +1,136 @@ +- name: kubernetes.pod + type: group + fields: + - name: start_time + type: date + description: | + Start time + - name: network + type: group + fields: + - name: rx + type: group + fields: + - name: bytes + type: long + format: bytes + unit: byte + metric_type: counter + description: | + Received bytes + - name: errors + type: long + metric_type: counter + description: | + Rx errors + - name: tx + type: group + fields: + - name: bytes + type: long + format: bytes + unit: byte + metric_type: counter + description: | + Transmitted bytes + - name: errors + type: long + metric_type: counter + description: | + Tx errors + - name: cpu + type: group + fields: + - name: usage + type: group + fields: + - name: nanocores + type: long + unit: byte + metric_type: gauge + description: | + CPU used nanocores + - name: node.pct + type: scaled_float + format: percent + unit: percent + metric_type: gauge + description: | + CPU usage as a percentage of the total node CPU + - name: limit.pct + type: scaled_float + format: percent + unit: percent + metric_type: gauge + description: | + CPU usage as a percentage of the defined limit for the pod containers (or total node CPU if one or more containers of the pod are unlimited) + - name: memory + type: group + fields: + - name: usage + type: group + fields: + - name: bytes + type: long + format: bytes + unit: byte + metric_type: gauge + description: | + Total memory usage + - name: node.pct + type: scaled_float + format: percent + unit: percent + metric_type: gauge + description: | + Memory usage as a percentage of the total node allocatable memory + - name: limit.pct + type: scaled_float + format: percent + unit: percent + metric_type: gauge + description: | + Memory usage as a percentage of the defined limit for the pod containers (or total node allocatable memory if unlimited) + - name: available + type: group + fields: + - name: bytes + type: long + format: bytes + unit: percent + metric_type: gauge + description: | + Total memory available + - name: working_set + type: group + fields: + - name: bytes + type: long + format: bytes + unit: percent + metric_type: gauge + description: | + Total working set memory + - name: rss + type: group + fields: + - name: bytes + type: long + format: bytes + unit: percent + metric_type: gauge + description: | + Total resident set size memory + - name: page_faults + type: long + metric_type: counter + description: | + Total page faults + - name: major_page_faults + type: long + metric_type: counter + description: | + Total major page faults + - name: ip + type: ip + description: Kubernetes pod IP diff --git a/test/packages/kubernetes/data_stream/pod/manifest.yml b/test/packages/kubernetes/data_stream/pod/manifest.yml new file mode 100644 index 0000000000..f9b3d340d8 --- /dev/null +++ b/test/packages/kubernetes/data_stream/pod/manifest.yml @@ -0,0 +1,43 @@ +title: Kubernetes Pod metrics +type: metrics +streams: + - input: kubernetes/metrics + vars: + - name: add_metadata + type: bool + title: Add Metadata + multi: false + required: true + show_user: true + default: true + - name: bearer_token_file + type: text + title: Bearer Token File + multi: false + required: true + show_user: true + default: /var/run/secrets/kubernetes.io/serviceaccount/token + - name: hosts + type: text + title: Hosts + multi: true + required: true + show_user: true + default: + - https://${env.NODE_NAME}:10250 + - name: period + type: text + title: Period + multi: false + required: true + show_user: true + default: 10s + - name: ssl.verification_mode + type: text + title: SSL Verification Mode + multi: false + required: true + show_user: true + default: none + title: Kubernetes Pod metrics + description: Collect Kubernetes Pod metrics diff --git a/test/packages/kubernetes/data_stream/pod/sample_event.json b/test/packages/kubernetes/data_stream/pod/sample_event.json new file mode 100644 index 0000000000..62e44029da --- /dev/null +++ b/test/packages/kubernetes/data_stream/pod/sample_event.json @@ -0,0 +1,125 @@ +{ + "@timestamp": "2020-06-25T12:34:59.729Z", + "kubernetes": { + "pod": { + "memory": { + "rss": { + "bytes": 7823360 + }, + "page_faults": 5742, + "major_page_faults": 0, + "usage": { + "limit": { + "pct": 0.0008033509820466402 + }, + "bytes": 13508608, + "node": { + "pct": 0.0008033509820466402 + } + }, + "available": { + "bytes": 0 + }, + "working_set": { + "bytes": 8556544 + } + }, + "network": { + "rx": { + "bytes": 25671624, + "errors": 0 + }, + "tx": { + "errors": 0, + "bytes": 1092900259 + } + }, + "start_time": "2020-06-18T11:12:58Z", + "name": "kube-state-metrics-57cd6fdf9-hd959", + "uid": "a7c61334-dd52-4a12-bed5-4daee4c74139", + "cpu": { + "usage": { + "nanocores": 2811918, + "node": { + "pct": 0.0007029795 + }, + "limit": { + "pct": 0.0007029795 + } + } + } + }, + "namespace": "kube-system", + "node": { + "name": "minikube" + } + }, + "event": { + "duration": 20735189, + "dataset": "kubernetes.pod", + "module": "kubernetes" + }, + "ecs": { + "version": "1.5.0" + }, + "metricset": { + "period": 10000, + "name": "pod" + }, + "service": { + "type": "kubernetes", + "address": "minikube:10250" + }, + "host": { + "name": "minikube", + "hostname": "minikube", + "architecture": "x86_64", + "os": { + "kernel": "4.19.81", + "codename": "Core", + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux" + }, + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false, + "ip": [ + "192.168.64.10", + "fe80::a883:2fff:fe7f:6b12", + "172.17.0.1", + "fe80::42:d4ff:fe8c:9493", + "fe80::2859:80ff:fe9e:fcd6", + "fe80::d83a:d9ff:fee9:7052", + "fe80::880a:b6ff:fe18:ba76", + "fe80::f447:faff:fe80:e88b", + "fe80::9cc3:ffff:fe95:e48e", + "fe80::6c1c:29ff:fe50:d40c", + "fe80::b4f3:11ff:fe60:14ed", + "fe80::20f2:2aff:fe96:1e7b", + "fe80::5434:baff:fede:5720", + "fe80::a878:91ff:fe29:81f7" + ], + "mac": [ + "aa:83:2f:7f:6b:12", + "02:42:d4:8c:94:93", + "2a:59:80:9e:fc:d6", + "da:3a:d9:e9:70:52", + "8a:0a:b6:18:ba:76", + "f6:47:fa:80:e8:8b", + "9e:c3:ff:95:e4:8e", + "6e:1c:29:50:d4:0c", + "b6:f3:11:60:14:ed", + "22:f2:2a:96:1e:7b", + "56:34:ba:de:57:20", + "aa:78:91:29:81:f7" + ] + }, + "agent": { + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "b964a246-96c0-456a-a5c2-8c8b1040ecaf", + "id": "f7ec69f9-4997-4e76-b6c7-0c75206b727a", + "name": "minikube" + } +} \ No newline at end of file diff --git a/test/packages/kubernetes/data_stream/proxy/_dev/deploy/k8s/.empty b/test/packages/kubernetes/data_stream/proxy/_dev/deploy/k8s/.empty new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/packages/kubernetes/data_stream/proxy/_dev/test/system/test-default-config.yml b/test/packages/kubernetes/data_stream/proxy/_dev/test/system/test-default-config.yml new file mode 100644 index 0000000000..d70f2cab74 --- /dev/null +++ b/test/packages/kubernetes/data_stream/proxy/_dev/test/system/test-default-config.yml @@ -0,0 +1,6 @@ +service: kubernetes +data_stream: + vars: + period: 5s + hosts: + - http://localhost:10249 diff --git a/test/packages/kubernetes/data_stream/proxy/agent/stream/stream.yml.hbs b/test/packages/kubernetes/data_stream/proxy/agent/stream/stream.yml.hbs new file mode 100644 index 0000000000..1723efeec0 --- /dev/null +++ b/test/packages/kubernetes/data_stream/proxy/agent/stream/stream.yml.hbs @@ -0,0 +1,6 @@ +metricsets: ["proxy"] +hosts: +{{#each hosts}} + - {{this}} +{{/each}} +period: {{period}} diff --git a/test/packages/kubernetes/data_stream/proxy/fields/agent.yml b/test/packages/kubernetes/data_stream/proxy/fields/agent.yml new file mode 100644 index 0000000000..da4e652c53 --- /dev/null +++ b/test/packages/kubernetes/data_stream/proxy/fields/agent.yml @@ -0,0 +1,198 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + diff --git a/test/packages/kubernetes/data_stream/proxy/fields/base-fields.yml b/test/packages/kubernetes/data_stream/proxy/fields/base-fields.yml new file mode 100644 index 0000000000..7c798f4534 --- /dev/null +++ b/test/packages/kubernetes/data_stream/proxy/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/test/packages/kubernetes/data_stream/proxy/fields/ecs.yml b/test/packages/kubernetes/data_stream/proxy/fields/ecs.yml new file mode 100644 index 0000000000..cd4e3a89b1 --- /dev/null +++ b/test/packages/kubernetes/data_stream/proxy/fields/ecs.yml @@ -0,0 +1,10 @@ +- external: ecs + name: ecs.version +- external: ecs + name: service.address +- external: ecs + name: service.type +- external: ecs + name: orchestrator.cluster.name +- external: ecs + name: orchestrator.cluster.url diff --git a/test/packages/kubernetes/data_stream/proxy/fields/fields.yml b/test/packages/kubernetes/data_stream/proxy/fields/fields.yml new file mode 100644 index 0000000000..7e37ae7893 --- /dev/null +++ b/test/packages/kubernetes/data_stream/proxy/fields/fields.yml @@ -0,0 +1,127 @@ +- name: kubernetes.proxy + type: group + fields: + - name: handler + dimension: true + type: keyword + description: | + Request handler + - name: code + dimension: true + type: keyword + description: | + HTTP code + - name: method + dimension: true + type: keyword + description: | + HTTP method + - name: host + dimension: true + type: keyword + description: | + Request host + - name: process + type: group + fields: + - name: cpu.sec + type: double + metric_type: counter + description: CPU seconds + - name: memory.resident.bytes + type: long + format: bytes + unit: byte + metric_type: gauge + description: Bytes in resident memory + - name: memory.virtual.bytes + type: long + format: bytes + unit: byte + metric_type: gauge + description: Bytes in virtual memory + - name: fds.open.count + type: long + metric_type: gauge + description: Number of open file descriptors + - name: started.sec + type: double + metric_type: gauge + description: Seconds since the process started + - name: http + type: group + fields: + - name: request.duration.us.percentile.* + type: object + description: Request duration microseconds percentiles + - name: request.duration.us.sum + type: double + metric_type: counter + unit: micros + description: Request duration microseconds cumulative sum + - name: request.duration.us.count + type: long + metric_type: counter + unit: micros + description: Request count for duration + - name: request.size.bytes.percentile.* + type: object + description: Request size percentiles + - name: request.size.bytes.sum + type: long + format: bytes + unit: byte + metric_type: counter + description: Request size cumulative sum + - name: request.size.bytes.count + type: long + unit: byte + metric_type: counter + description: Request count for size + - name: response.size.bytes.percentile.* + type: object + description: Response size percentiles + - name: response.size.bytes.sum + type: long + format: bytes + unit: byte + metric_type: counter + description: Response size cumulative sum + - name: response.size.bytes.count + type: long + metric_type: counter + description: Response count + - name: request.count + type: long + metric_type: counter + description: Request count + - name: client.request.count + type: long + metric_type: counter + description: | + Number of requests as client + - name: sync + type: group + fields: + - name: rules.duration.us.sum + type: long + metric_type: counter + description: SyncProxyRules duration, sum of durations in microseconds + - name: rules.duration.us.count + type: long + metric_type: counter + description: SyncProxyRules duration, number of operations + - name: rules.duration.us.bucket.* + type: object + description: SyncProxyRules duration, histogram buckets + - name: networkprogramming.duration.us.sum + type: long + metric_type: counter + description: Network programming duration, sum in microseconds + - name: networkprogramming.duration.us.count + type: long + metric_type: counter + description: Network programming duration, number of operations + - name: networkprogramming.duration.us.bucket.* + type: object + description: Network programming duration, histogram buckets diff --git a/test/packages/kubernetes/data_stream/proxy/manifest.yml b/test/packages/kubernetes/data_stream/proxy/manifest.yml new file mode 100644 index 0000000000..8b4f0d944f --- /dev/null +++ b/test/packages/kubernetes/data_stream/proxy/manifest.yml @@ -0,0 +1,22 @@ +title: Kubernetes Proxy metrics +type: metrics +streams: + - input: kubernetes/metrics + vars: + - name: hosts + type: text + title: Hosts + multi: true + required: true + show_user: true + default: + - localhost:10249 + - name: period + type: text + title: Period + multi: false + required: true + show_user: true + default: 10s + title: Kubernetes Proxy metrics + description: Collect Kubernetes Proxy metrics diff --git a/test/packages/kubernetes/data_stream/proxy/sample_event.json b/test/packages/kubernetes/data_stream/proxy/sample_event.json new file mode 100644 index 0000000000..50ba492ea8 --- /dev/null +++ b/test/packages/kubernetes/data_stream/proxy/sample_event.json @@ -0,0 +1,216 @@ +{ + "@timestamp": "2020-06-25T12:35:29.639Z", + "agent": { + "name": "minikube", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "b964a246-96c0-456a-a5c2-8c8b1040ecaf", + "id": "f7ec69f9-4997-4e76-b6c7-0c75206b727a" + }, + "host": { + "ip": [ + "192.168.64.10", + "fe80::a883:2fff:fe7f:6b12", + "172.17.0.1", + "fe80::42:d4ff:fe8c:9493", + "fe80::2859:80ff:fe9e:fcd6", + "fe80::d83a:d9ff:fee9:7052", + "fe80::880a:b6ff:fe18:ba76", + "fe80::f447:faff:fe80:e88b", + "fe80::9cc3:ffff:fe95:e48e", + "fe80::6c1c:29ff:fe50:d40c", + "fe80::b4f3:11ff:fe60:14ed", + "fe80::20f2:2aff:fe96:1e7b", + "fe80::5434:baff:fede:5720", + "fe80::a878:91ff:fe29:81f7" + ], + "name": "minikube", + "mac": [ + "aa:83:2f:7f:6b:12", + "02:42:d4:8c:94:93", + "2a:59:80:9e:fc:d6", + "da:3a:d9:e9:70:52", + "8a:0a:b6:18:ba:76", + "f6:47:fa:80:e8:8b", + "9e:c3:ff:95:e4:8e", + "6e:1c:29:50:d4:0c", + "b6:f3:11:60:14:ed", + "22:f2:2a:96:1e:7b", + "56:34:ba:de:57:20", + "aa:78:91:29:81:f7" + ], + "hostname": "minikube", + "architecture": "x86_64", + "os": { + "codename": "Core", + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81" + }, + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false + }, + "kubernetes": { + "proxy": { + "sync": { + "rules": { + "duration": { + "us": { + "sum": 763620.9329999998, + "count": 18, + "bucket": { + "1000": 0, + "2000": 0, + "4000": 0, + "8000": 0, + "16000": 0, + "32000": 10, + "64000": 16, + "128000": 17, + "256000": 18, + "512000": 18, + "1024000": 18, + "2048000": 18, + "4096000": 18, + "8192000": 18, + "16384000": 18, + "+Inf": 18 + } + } + } + }, + "networkprogramming": { + "duration": { + "us": { + "count": 19, + "bucket": { + "0": 0, + "250000": 4, + "500000": 8, + "1000000": 11, + "2000000": 11, + "3000000": 11, + "4000000": 11, + "5000000": 11, + "6000000": 11, + "7000000": 11, + "8000000": 11, + "9000000": 11, + "10000000": 11, + "11000000": 11, + "12000000": 11, + "13000000": 11, + "14000000": 11, + "15000000": 11, + "16000000": 11, + "17000000": 11, + "18000000": 11, + "19000000": 11, + "20000000": 11, + "21000000": 11, + "22000000": 11, + "23000000": 11, + "24000000": 11, + "25000000": 11, + "26000000": 11, + "27000000": 11, + "28000000": 11, + "29000000": 11, + "30000000": 11, + "31000000": 11, + "32000000": 11, + "33000000": 11, + "34000000": 11, + "35000000": 11, + "36000000": 11, + "37000000": 11, + "38000000": 11, + "39000000": 11, + "40000000": 11, + "41000000": 11, + "42000000": 11, + "43000000": 11, + "44000000": 11, + "45000000": 11, + "46000000": 11, + "47000000": 11, + "48000000": 11, + "49000000": 11, + "50000000": 11, + "51000000": 11, + "52000000": 11, + "53000000": 11, + "54000000": 11, + "55000000": 11, + "56000000": 11, + "57000000": 11, + "58000000": 11, + "59000000": 11, + "60000000": 11, + "65000000": 11, + "70000000": 11, + "75000000": 11, + "80000000": 11, + "85000000": 11, + "90000000": 11, + "95000000": 11, + "100000000": 11, + "105000000": 11, + "110000000": 11, + "115000000": 11, + "120000000": 11, + "150000000": 11, + "180000000": 11, + "210000000": 11, + "240000000": 11, + "270000000": 11, + "300000000": 11, + "+Inf": 19 + }, + "sum": 5571080914163.27 + } + } + } + }, + "process": { + "cpu": { + "sec": 8 + }, + "memory": { + "resident": { + "bytes": 37609472 + }, + "virtual": { + "bytes": 143990784 + } + }, + "started": { + "sec": 1593069580.69 + }, + "fds": { + "open": { + "count": 17 + } + } + } + } + }, + "ecs": { + "version": "1.5.0" + }, + "event": { + "module": "kubernetes", + "duration": 2031254, + "dataset": "kubernetes.proxy" + }, + "metricset": { + "name": "proxy", + "period": 10000 + }, + "service": { + "address": "localhost:10249", + "type": "kubernetes" + } +} \ No newline at end of file diff --git a/test/packages/kubernetes/data_stream/scheduler/_dev/deploy/k8s/.empty b/test/packages/kubernetes/data_stream/scheduler/_dev/deploy/k8s/.empty new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/packages/kubernetes/data_stream/scheduler/_dev/test/system/test-default-config.yml b/test/packages/kubernetes/data_stream/scheduler/_dev/test/system/test-default-config.yml new file mode 100644 index 0000000000..c775d296e6 --- /dev/null +++ b/test/packages/kubernetes/data_stream/scheduler/_dev/test/system/test-default-config.yml @@ -0,0 +1,8 @@ +service: kubernetes +data_stream: + vars: + period: 5s + hosts: + - https://0.0.0.0:10259 + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + ssl.verification_mode: "none" diff --git a/test/packages/kubernetes/data_stream/scheduler/agent/stream/stream.yml.hbs b/test/packages/kubernetes/data_stream/scheduler/agent/stream/stream.yml.hbs new file mode 100644 index 0000000000..1c34b2b902 --- /dev/null +++ b/test/packages/kubernetes/data_stream/scheduler/agent/stream/stream.yml.hbs @@ -0,0 +1,12 @@ +metricsets: ["scheduler"] +hosts: +{{#each hosts}} + - {{this}} +{{/each}} +period: {{period}} + +{{#if bearer_token_file}} +bearer_token_file: {{bearer_token_file}} +ssl.verification_mode: {{ssl.verification_mode}} +{{/if}} +condition: ${kubernetes.labels.{{~scheduler_label_key~}} } == '{{scheduler_label_value}}' \ No newline at end of file diff --git a/test/packages/kubernetes/data_stream/scheduler/fields/agent.yml b/test/packages/kubernetes/data_stream/scheduler/fields/agent.yml new file mode 100644 index 0000000000..da4e652c53 --- /dev/null +++ b/test/packages/kubernetes/data_stream/scheduler/fields/agent.yml @@ -0,0 +1,198 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + diff --git a/test/packages/kubernetes/data_stream/scheduler/fields/base-fields.yml b/test/packages/kubernetes/data_stream/scheduler/fields/base-fields.yml new file mode 100644 index 0000000000..d43ffd7964 --- /dev/null +++ b/test/packages/kubernetes/data_stream/scheduler/fields/base-fields.yml @@ -0,0 +1,90 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. +- name: kubernetes + type: group + fields: + - name: pod.name + type: keyword + description: > + Kubernetes pod name + + - name: pod.uid + type: keyword + description: > + Kubernetes pod UID + + - name: pod.ip + type: ip + description: > + Kubernetes pod IP + + - name: namespace + type: keyword + description: > + Kubernetes namespace + + - name: node.name + type: keyword + description: > + Kubernetes node name + + - name: node.hostname + type: keyword + description: > + Kubernetes hostname as reported by the node’s kernel + + - name: labels.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes labels map + + - name: annotations.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes annotations map + + - name: selectors.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes Service selectors map + + - name: replicaset.name + type: keyword + description: > + Kubernetes replicaset name + + - name: deployment.name + type: keyword + description: > + Kubernetes deployment name + + - name: statefulset.name + type: keyword + description: > + Kubernetes statefulset name + + - name: container.name + type: keyword + description: > + Kubernetes container name + + - name: container.image + type: keyword + description: >- + Kubernetes container image diff --git a/test/packages/kubernetes/data_stream/scheduler/fields/ecs.yml b/test/packages/kubernetes/data_stream/scheduler/fields/ecs.yml new file mode 100644 index 0000000000..cd4e3a89b1 --- /dev/null +++ b/test/packages/kubernetes/data_stream/scheduler/fields/ecs.yml @@ -0,0 +1,10 @@ +- external: ecs + name: ecs.version +- external: ecs + name: service.address +- external: ecs + name: service.type +- external: ecs + name: orchestrator.cluster.name +- external: ecs + name: orchestrator.cluster.url diff --git a/test/packages/kubernetes/data_stream/scheduler/fields/fields.yml b/test/packages/kubernetes/data_stream/scheduler/fields/fields.yml new file mode 100644 index 0000000000..fa717504af --- /dev/null +++ b/test/packages/kubernetes/data_stream/scheduler/fields/fields.yml @@ -0,0 +1,163 @@ +- name: kubernetes.scheduler + type: group + fields: + - name: handler + dimension: true + type: keyword + description: | + Request handler + - name: code + dimension: true + type: keyword + description: | + HTTP code + - name: method + dimension: true + type: keyword + description: | + HTTP method + - name: host + dimension: true + type: keyword + description: | + Request host + - name: name + dimension: true + type: keyword + description: | + Name for the resource + - name: result + dimension: true + type: keyword + description: | + Schedule attempt result + - name: operation + dimension: true + type: keyword + description: | + Scheduling operation + - name: process + type: group + fields: + - name: cpu.sec + type: double + metric_type: counter + description: CPU seconds + - name: memory.resident.bytes + type: long + format: bytes + unit: byte + metric_type: gauge + description: Bytes in resident memory + - name: memory.virtual.bytes + type: long + format: bytes + unit: byte + metric_type: gauge + description: Bytes in virtual memory + - name: fds.open.count + type: long + metric_type: gauge + description: Number of open file descriptors + - name: started.sec + type: double + metric_type: gauge + description: Seconds since the process started + - name: http + type: group + fields: + - name: request.duration.us.percentile.* + type: object + description: Request duration microseconds percentiles + - name: request.duration.us.sum + type: double + metric_type: counter + unit: micros + description: Request duration microseconds cumulative sum + - name: request.duration.us.count + type: long + metric_type: counter + unit: micros + description: Request count for duration + - name: request.size.bytes.percentile.* + type: object + description: Request size percentiles + - name: request.size.bytes.sum + type: long + format: bytes + unit: byte + metric_type: counter + description: Request size cumulative sum + - name: request.size.bytes.count + type: long + unit: byte + metric_type: counter + description: Request count for size + - name: response.size.bytes.percentile.* + type: object + description: Response size percentiles + - name: response.size.bytes.sum + type: long + format: bytes + unit: byte + metric_type: counter + description: Response size cumulative sum + - name: response.size.bytes.count + type: long + metric_type: counter + description: Response count + - name: request.count + type: long + metric_type: counter + description: Request count + - name: client.request.count + type: long + metric_type: counter + description: | + Number of requests as client + - name: leader.is_master + type: boolean + description: | + Whether the node is master + - name: scheduling + type: group + fields: + - name: e2e.duration.us.bucket.* + type: object + description: End to end scheduling duration microseconds + - name: e2e.duration.us.sum + type: long + unit: micros + metric_type: counter + description: End to end scheduling duration microseconds sum + - name: e2e.duration.us.count + type: long + unit: micros + metric_type: counter + description: End to end scheduling count + - name: pod.preemption.victims.bucket.* + type: long + description: Pod preemption victims + - name: pod.preemption.victims.sum + type: long + metric_type: counter + description: Pod preemption victims sum + - name: pod.preemption.victims.count + type: long + metric_type: counter + description: Pod preemption victims count + - name: pod.attempts.count + type: long + metric_type: counter + description: Pod attempts count + - name: duration.seconds.percentile.* + type: object + description: Scheduling duration percentiles + - name: duration.seconds.sum + type: double + metric_type: counter + description: Scheduling duration cumulative sum + - name: duration.seconds.count + type: long + metric_type: counter + description: Scheduling count diff --git a/test/packages/kubernetes/data_stream/scheduler/manifest.yml b/test/packages/kubernetes/data_stream/scheduler/manifest.yml new file mode 100644 index 0000000000..cd7e4c078b --- /dev/null +++ b/test/packages/kubernetes/data_stream/scheduler/manifest.yml @@ -0,0 +1,51 @@ +title: Kubernetes Scheduler metrics +type: metrics +streams: + - input: kubernetes/metrics + enabled: false + vars: + - name: bearer_token_file + type: text + title: Bearer Token File + multi: false + required: true + show_user: true + default: /var/run/secrets/kubernetes.io/serviceaccount/token + - name: hosts + type: text + title: Hosts + multi: true + required: true + show_user: true + default: + - https://0.0.0.0:10259 + - name: period + type: text + title: Period + multi: false + required: true + show_user: true + default: 10s + - name: ssl.verification_mode + type: text + title: SSL Verification Mode + multi: false + required: true + show_user: true + default: none + - name: scheduler_label_key + type: text + title: Kubernetes Scheduler Label key + multi: false + required: true + show_user: false + default: component + - name: scheduler_label_value + type: text + title: Kubernetes Scheduler Label value + multi: false + required: true + show_user: false + default: kube-scheduler + title: Kubernetes Scheduler metrics + description: Collect Kubernetes Scheduler metrics diff --git a/test/packages/kubernetes/data_stream/scheduler/sample_event.json b/test/packages/kubernetes/data_stream/scheduler/sample_event.json new file mode 100644 index 0000000000..b932b1797d --- /dev/null +++ b/test/packages/kubernetes/data_stream/scheduler/sample_event.json @@ -0,0 +1,79 @@ +{ + "@timestamp": "2020-06-25T12:35:59.624Z", + "agent": { + "version": "8.0.0", + "ephemeral_id": "b964a246-96c0-456a-a5c2-8c8b1040ecaf", + "id": "f7ec69f9-4997-4e76-b6c7-0c75206b727a", + "name": "minikube", + "type": "metricbeat" + }, + "host": { + "hostname": "minikube", + "architecture": "x86_64", + "os": { + "kernel": "4.19.81", + "codename": "Core", + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux" + }, + "name": "minikube", + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false, + "ip": [ + "192.168.64.10", + "fe80::a883:2fff:fe7f:6b12", + "172.17.0.1", + "fe80::42:d4ff:fe8c:9493", + "fe80::2859:80ff:fe9e:fcd6", + "fe80::d83a:d9ff:fee9:7052", + "fe80::880a:b6ff:fe18:ba76", + "fe80::f447:faff:fe80:e88b", + "fe80::9cc3:ffff:fe95:e48e", + "fe80::6c1c:29ff:fe50:d40c", + "fe80::b4f3:11ff:fe60:14ed", + "fe80::20f2:2aff:fe96:1e7b", + "fe80::5434:baff:fede:5720", + "fe80::a878:91ff:fe29:81f7" + ], + "mac": [ + "aa:83:2f:7f:6b:12", + "02:42:d4:8c:94:93", + "2a:59:80:9e:fc:d6", + "da:3a:d9:e9:70:52", + "8a:0a:b6:18:ba:76", + "f6:47:fa:80:e8:8b", + "9e:c3:ff:95:e4:8e", + "6e:1c:29:50:d4:0c", + "b6:f3:11:60:14:ed", + "22:f2:2a:96:1e:7b", + "56:34:ba:de:57:20", + "aa:78:91:29:81:f7" + ] + }, + "ecs": { + "version": "1.5.0" + }, + "event": { + "duration": 7245648, + "dataset": "kubernetes.scheduler", + "module": "kubernetes" + }, + "metricset": { + "name": "scheduler", + "period": 10000 + }, + "service": { + "address": "localhost:10251", + "type": "kubernetes" + }, + "kubernetes": { + "scheduler": { + "name": "kube-scheduler", + "leader": { + "is_master": true + } + } + } +} \ No newline at end of file diff --git a/test/packages/kubernetes/data_stream/state_container/_dev/test/system/test-default-config.yml b/test/packages/kubernetes/data_stream/state_container/_dev/test/system/test-default-config.yml new file mode 100644 index 0000000000..0f4bd620c0 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_container/_dev/test/system/test-default-config.yml @@ -0,0 +1,6 @@ +service: kubernetes +data_stream: + vars: + hosts: + # this is the DNS name of the k8s service for kube-state-metrics deployment + - http://kube-state-metrics:8080 diff --git a/test/packages/kubernetes/data_stream/state_container/agent/stream/stream.yml.hbs b/test/packages/kubernetes/data_stream/state_container/agent/stream/stream.yml.hbs new file mode 100644 index 0000000000..55bf95929d --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_container/agent/stream/stream.yml.hbs @@ -0,0 +1,10 @@ +metricsets: ["state_container"] +add_metadata: {{add_metadata}} +hosts: +{{#each hosts}} + - {{this}} +{{/each}} +period: {{period}} +{{#if leaderelection}} +condition: ${kubernetes_leaderelection.leader} == true +{{/if}} diff --git a/test/packages/kubernetes/data_stream/state_container/fields/agent.yml b/test/packages/kubernetes/data_stream/state_container/fields/agent.yml new file mode 100644 index 0000000000..d16c882552 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_container/fields/agent.yml @@ -0,0 +1,199 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + dimension: true + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + diff --git a/test/packages/kubernetes/data_stream/state_container/fields/base-fields.yml b/test/packages/kubernetes/data_stream/state_container/fields/base-fields.yml new file mode 100644 index 0000000000..98c09161d6 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_container/fields/base-fields.yml @@ -0,0 +1,98 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. +- name: kubernetes + type: group + fields: + - name: pod.name + dimension: true + type: keyword + description: > + Kubernetes pod name + + - name: pod.uid + dimension: true + type: keyword + description: > + Kubernetes pod UID + + - name: pod.ip + type: ip + description: > + Kubernetes pod IP + + - name: namespace + type: keyword + description: > + Kubernetes namespace + + - name: node.name + type: keyword + description: > + Kubernetes node name + + - name: node.hostname + type: keyword + description: > + Kubernetes hostname as reported by the node’s kernel + + - name: labels.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes labels map + + - name: annotations.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes annotations map + + - name: selectors.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes Service selectors map + + - name: replicaset.name + type: keyword + description: > + Kubernetes replicaset name + + - name: deployment.name + type: keyword + description: > + Kubernetes deployment name + + - name: daemonset.name + type: keyword + description: > + Kubernetes daemonset name + + - name: statefulset.name + type: keyword + description: > + Kubernetes statefulset name + + - name: container.name + dimension: true + type: keyword + description: > + Kubernetes container name + + - name: container.image + type: keyword + description: >- + Kubernetes container image diff --git a/test/packages/kubernetes/data_stream/state_container/fields/ecs.yml b/test/packages/kubernetes/data_stream/state_container/fields/ecs.yml new file mode 100644 index 0000000000..31cb2817b8 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_container/fields/ecs.yml @@ -0,0 +1,12 @@ +- external: ecs + name: ecs.version +- external: ecs + name: service.address +- external: ecs + name: service.type +- external: ecs + name: container.runtime +- external: ecs + name: orchestrator.cluster.name +- external: ecs + name: orchestrator.cluster.url diff --git a/test/packages/kubernetes/data_stream/state_container/fields/fields.yml b/test/packages/kubernetes/data_stream/state_container/fields/fields.yml new file mode 100644 index 0000000000..858f894d18 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_container/fields/fields.yml @@ -0,0 +1,66 @@ +- name: kubernetes.container + type: group + fields: + - name: id + type: keyword + description: Container id + - name: status + type: group + fields: + - name: phase + type: keyword + description: | + Container phase (running, waiting, terminated) + - name: ready + type: boolean + description: | + Container ready status + - name: restarts + type: integer + metric_type: counter + description: | + Container restarts count + - name: reason + type: keyword + description: | + Waiting (ContainerCreating, CrashLoopBackoff, ErrImagePull, ImagePullBackoff) or termination (Completed, ContainerCannotRun, Error, OOMKilled) reason. + - name: cpu + type: group + fields: + - name: limit.cores + type: float + metric_type: gauge + description: | + Container CPU cores limit + - name: request.cores + type: float + metric_type: gauge + description: | + Container CPU requested cores + - name: limit.nanocores + type: long + metric_type: gauge + description: | + Container CPU nanocores limit + - name: request.nanocores + type: long + metric_type: gauge + description: | + Container CPU requested nanocores + - name: memory + type: group + fields: + - name: limit.bytes + type: long + format: bytes + unit: byte + metric_type: gauge + description: | + Container memory limit in bytes + - name: request.bytes + type: long + format: bytes + unit: byte + metric_type: gauge + description: | + Container requested memory in bytes diff --git a/test/packages/kubernetes/data_stream/state_container/manifest.yml b/test/packages/kubernetes/data_stream/state_container/manifest.yml new file mode 100644 index 0000000000..a51516b6e3 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_container/manifest.yml @@ -0,0 +1,37 @@ +title: Kubernetes Container metrics +type: metrics +streams: + - input: kubernetes/metrics + enabled: false + vars: + - name: add_metadata + type: bool + title: Add Metadata + multi: false + required: true + show_user: true + default: true + - name: hosts + type: text + title: Hosts + multi: true + required: true + show_user: true + default: + - kube-state-metrics:8080 + - name: leaderelection + type: bool + title: Leader Election + multi: false + required: true + show_user: true + default: true + - name: period + type: text + title: Period + multi: false + required: true + show_user: true + default: 10s + title: Kubernetes Container metrics + description: Collect Kubernetes Container metrics from kube_state_metrics diff --git a/test/packages/kubernetes/data_stream/state_container/sample_event.json b/test/packages/kubernetes/data_stream/state_container/sample_event.json new file mode 100644 index 0000000000..fda9be9310 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_container/sample_event.json @@ -0,0 +1,80 @@ +{ + "@timestamp": "2020-06-25T12:36:34.469Z", + "host": { + "ip": [ + "172.17.0.11" + ], + "mac": [ + "02:42:ac:11:00:0b" + ], + "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "architecture": "x86_64", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "os": { + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81", + "codename": "Core", + "platform": "centos", + "version": "7 (Core)" + }, + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false + }, + "event": { + "dataset": "kubernetes.container", + "module": "kubernetes", + "duration": 8554499 + }, + "kubernetes": { + "node": { + "name": "minikube" + }, + "labels": { + "component": "kube-scheduler", + "tier": "control-plane" + }, + "container": { + "image": "k8s.gcr.io/kube-scheduler:v1.17.0", + "name": "kube-scheduler", + "cpu": { + "request": { + "cores": 0.1 + } + }, + "status": { + "phase": "running", + "ready": true, + "restarts": 10 + }, + "id": "docker://b00b185f2b304a7ece804d1af28eb232f825255f716bcc85ef5bd20d5a4f45d4" + }, + "pod": { + "name": "kube-scheduler-minikube", + "uid": "9cdbd5ea-7638-4e86-a706-a5b222d86f26" + }, + "namespace": "kube-system" + }, + "ecs": { + "version": "1.5.0" + }, + "agent": { + "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", + "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "type": "metricbeat", + "version": "8.0.0" + }, + "container": { + "runtime": "docker", + "id": "b00b185f2b304a7ece804d1af28eb232f825255f716bcc85ef5bd20d5a4f45d4" + }, + "service": { + "address": "kube-state-metrics:8080", + "type": "kubernetes" + }, + "metricset": { + "name": "state_container", + "period": 10000 + } +} \ No newline at end of file diff --git a/test/packages/kubernetes/data_stream/state_cronjob/_dev/test/system/test-default-config.yml b/test/packages/kubernetes/data_stream/state_cronjob/_dev/test/system/test-default-config.yml new file mode 100644 index 0000000000..0f4bd620c0 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_cronjob/_dev/test/system/test-default-config.yml @@ -0,0 +1,6 @@ +service: kubernetes +data_stream: + vars: + hosts: + # this is the DNS name of the k8s service for kube-state-metrics deployment + - http://kube-state-metrics:8080 diff --git a/test/packages/kubernetes/data_stream/state_cronjob/agent/stream/stream.yml.hbs b/test/packages/kubernetes/data_stream/state_cronjob/agent/stream/stream.yml.hbs new file mode 100644 index 0000000000..08b16b3392 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_cronjob/agent/stream/stream.yml.hbs @@ -0,0 +1,11 @@ +metricsets: ["state_cronjob"] +add_metadata: {{add_metadata}} +hosts: +{{#each hosts}} + - {{this}} +{{/each}} +period: {{period}} +{{#if leaderelection}} +condition: ${kubernetes_leaderelection.leader} == true +{{/if}} + diff --git a/test/packages/kubernetes/data_stream/state_cronjob/fields/agent.yml b/test/packages/kubernetes/data_stream/state_cronjob/fields/agent.yml new file mode 100644 index 0000000000..da4e652c53 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_cronjob/fields/agent.yml @@ -0,0 +1,198 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + diff --git a/test/packages/kubernetes/data_stream/state_cronjob/fields/base-fields.yml b/test/packages/kubernetes/data_stream/state_cronjob/fields/base-fields.yml new file mode 100644 index 0000000000..331f87f888 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_cronjob/fields/base-fields.yml @@ -0,0 +1,93 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. +- name: kubernetes + type: group + fields: + - name: pod.name + dimension: true + type: keyword + description: > + Kubernetes pod name + + - name: pod.uid + dimension: true + type: keyword + description: > + Kubernetes pod UID + + - name: pod.ip + type: ip + description: > + Kubernetes pod IP + + - name: namespace + type: keyword + description: > + Kubernetes namespace + + - name: node.name + type: keyword + description: > + Kubernetes node name + + - name: node.hostname + type: keyword + description: > + Kubernetes hostname as reported by the node’s kernel + + - name: labels.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes labels map + + - name: annotations.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes annotations map + + - name: selectors.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes Service selectors map + + - name: replicaset.name + type: keyword + description: > + Kubernetes replicaset name + + - name: deployment.name + type: keyword + description: > + Kubernetes deployment name + + - name: statefulset.name + type: keyword + description: > + Kubernetes statefulset name + + - name: container.name + dimension: true + type: keyword + description: > + Kubernetes container name + + - name: container.image + type: keyword + description: >- + Kubernetes container image diff --git a/test/packages/kubernetes/data_stream/state_cronjob/fields/ecs.yml b/test/packages/kubernetes/data_stream/state_cronjob/fields/ecs.yml new file mode 100644 index 0000000000..cd4e3a89b1 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_cronjob/fields/ecs.yml @@ -0,0 +1,10 @@ +- external: ecs + name: ecs.version +- external: ecs + name: service.address +- external: ecs + name: service.type +- external: ecs + name: orchestrator.cluster.name +- external: ecs + name: orchestrator.cluster.url diff --git a/test/packages/kubernetes/data_stream/state_cronjob/fields/fields.yml b/test/packages/kubernetes/data_stream/state_cronjob/fields/fields.yml new file mode 100644 index 0000000000..5e3470530c --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_cronjob/fields/fields.yml @@ -0,0 +1,43 @@ +- name: kubernetes.cronjob + type: group + fields: + - name: name + dimension: true + type: keyword + description: Cronjob name + - name: schedule + dimension: true + type: keyword + description: Cronjob schedule + - name: concurrency + dimension: true + type: keyword + description: Concurrency policy + - name: active.count + type: long + metric_type: gauge + description: Number of active pods for the cronjob + - name: is_suspended + dimension: true + type: boolean + description: Whether the cronjob is suspended + - name: created.sec + type: double + unit: s + metric_type: gauge + description: Epoch seconds since the cronjob was created + - name: last_schedule.sec + type: double + unit: s + metric_type: gauge + description: Epoch seconds for last cronjob run + - name: next_schedule.sec + type: double + unit: s + metric_type: gauge + description: Epoch seconds for next cronjob run + - name: deadline.sec + type: long + unit: s + metric_type: gauge + description: Deadline seconds after schedule for considering failed diff --git a/test/packages/kubernetes/data_stream/state_cronjob/manifest.yml b/test/packages/kubernetes/data_stream/state_cronjob/manifest.yml new file mode 100644 index 0000000000..b13c06ae26 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_cronjob/manifest.yml @@ -0,0 +1,37 @@ +title: Kubernetes Cronjob metrics +type: metrics +streams: + - input: kubernetes/metrics + enabled: false + vars: + - name: add_metadata + type: bool + title: Add Metadata + multi: false + required: true + show_user: true + default: true + - name: hosts + type: text + title: Hosts + multi: true + required: true + show_user: true + default: + - kube-state-metrics:8080 + - name: leaderelection + type: bool + title: Leader Election + multi: false + required: true + show_user: true + default: true + - name: period + type: text + title: Period + multi: false + required: true + show_user: true + default: 10s + title: Kubernetes Cronjob metrics + description: Collect Kubernetes Cronjob metrics from kube_state_metrics diff --git a/test/packages/kubernetes/data_stream/state_cronjob/sample_event.json b/test/packages/kubernetes/data_stream/state_cronjob/sample_event.json new file mode 100644 index 0000000000..f297190ba7 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_cronjob/sample_event.json @@ -0,0 +1,66 @@ +{ + "@timestamp": "2020-06-25T12:43:04.384Z", + "metricset": { + "name": "state_cronjob", + "period": 10000 + }, + "service": { + "address": "kube-state-metrics:8080", + "type": "kubernetes" + }, + "ecs": { + "version": "1.5.0" + }, + "host": { + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false, + "ip": [ + "172.17.0.11" + ], + "mac": [ + "02:42:ac:11:00:0b" + ], + "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "architecture": "x86_64", + "os": { + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81", + "codename": "Core", + "platform": "centos", + "version": "7 (Core)" + } + }, + "event": { + "dataset": "kubernetes.cronjob", + "module": "kubernetes", + "duration": 9482053 + }, + "kubernetes": { + "namespace": "default", + "cronjob": { + "active": { + "count": 0 + }, + "is_suspended": false, + "name": "hello", + "next_schedule": { + "sec": 1593088980 + }, + "last_schedule": { + "sec": 1593088920 + }, + "created": { + "sec": 1593088862 + } + } + }, + "agent": { + "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", + "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "type": "metricbeat", + "version": "8.0.0" + } +} \ No newline at end of file diff --git a/test/packages/kubernetes/data_stream/state_daemonset/_dev/test/system/test-default-config.yml b/test/packages/kubernetes/data_stream/state_daemonset/_dev/test/system/test-default-config.yml new file mode 100644 index 0000000000..0f4bd620c0 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_daemonset/_dev/test/system/test-default-config.yml @@ -0,0 +1,6 @@ +service: kubernetes +data_stream: + vars: + hosts: + # this is the DNS name of the k8s service for kube-state-metrics deployment + - http://kube-state-metrics:8080 diff --git a/test/packages/kubernetes/data_stream/state_daemonset/agent/stream/stream.yml.hbs b/test/packages/kubernetes/data_stream/state_daemonset/agent/stream/stream.yml.hbs new file mode 100644 index 0000000000..f7b98418b7 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_daemonset/agent/stream/stream.yml.hbs @@ -0,0 +1,10 @@ +metricsets: ["state_daemonset"] +add_metadata: {{add_metadata}} +hosts: +{{#each hosts}} + - {{this}} +{{/each}} +period: {{period}} +{{#if leaderelection}} +condition: ${kubernetes_leaderelection.leader} == true +{{/if}} diff --git a/test/packages/kubernetes/data_stream/state_daemonset/fields/agent.yml b/test/packages/kubernetes/data_stream/state_daemonset/fields/agent.yml new file mode 100644 index 0000000000..da4e652c53 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_daemonset/fields/agent.yml @@ -0,0 +1,198 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + diff --git a/test/packages/kubernetes/data_stream/state_daemonset/fields/base-fields.yml b/test/packages/kubernetes/data_stream/state_daemonset/fields/base-fields.yml new file mode 100644 index 0000000000..d43ffd7964 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_daemonset/fields/base-fields.yml @@ -0,0 +1,90 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. +- name: kubernetes + type: group + fields: + - name: pod.name + type: keyword + description: > + Kubernetes pod name + + - name: pod.uid + type: keyword + description: > + Kubernetes pod UID + + - name: pod.ip + type: ip + description: > + Kubernetes pod IP + + - name: namespace + type: keyword + description: > + Kubernetes namespace + + - name: node.name + type: keyword + description: > + Kubernetes node name + + - name: node.hostname + type: keyword + description: > + Kubernetes hostname as reported by the node’s kernel + + - name: labels.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes labels map + + - name: annotations.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes annotations map + + - name: selectors.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes Service selectors map + + - name: replicaset.name + type: keyword + description: > + Kubernetes replicaset name + + - name: deployment.name + type: keyword + description: > + Kubernetes deployment name + + - name: statefulset.name + type: keyword + description: > + Kubernetes statefulset name + + - name: container.name + type: keyword + description: > + Kubernetes container name + + - name: container.image + type: keyword + description: >- + Kubernetes container image diff --git a/test/packages/kubernetes/data_stream/state_daemonset/fields/ecs.yml b/test/packages/kubernetes/data_stream/state_daemonset/fields/ecs.yml new file mode 100644 index 0000000000..cd4e3a89b1 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_daemonset/fields/ecs.yml @@ -0,0 +1,10 @@ +- external: ecs + name: ecs.version +- external: ecs + name: service.address +- external: ecs + name: service.type +- external: ecs + name: orchestrator.cluster.name +- external: ecs + name: orchestrator.cluster.url diff --git a/test/packages/kubernetes/data_stream/state_daemonset/fields/fields.yml b/test/packages/kubernetes/data_stream/state_daemonset/fields/fields.yml new file mode 100644 index 0000000000..c763091832 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_daemonset/fields/fields.yml @@ -0,0 +1,31 @@ +- name: kubernetes.daemonset + type: group + fields: + - name: name + dimension: true + type: keyword + - name: replicas + type: group + description: | + Kubernetes DaemonSet replica metrics + fields: + - name: available + type: long + metric_type: gauge + description: | + The number of available replicas per DaemonSet + - name: desired + type: long + metric_type: gauge + description: | + The desired number of replicas per DaemonSet + - name: ready + type: long + metric_type: gauge + description: | + The number of ready replicas per DaemonSet + - name: unavailable + type: long + metric_type: gauge + description: | + The number of unavailable replicas per DaemonSet diff --git a/test/packages/kubernetes/data_stream/state_daemonset/manifest.yml b/test/packages/kubernetes/data_stream/state_daemonset/manifest.yml new file mode 100644 index 0000000000..e7a1c826f3 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_daemonset/manifest.yml @@ -0,0 +1,37 @@ +title: Kubernetes Deamonset metrics +type: metrics +streams: + - input: kubernetes/metrics + enabled: false + vars: + - name: add_metadata + type: bool + title: Add Metadata + multi: false + required: true + show_user: true + default: true + - name: hosts + type: text + title: Hosts + multi: true + required: true + show_user: true + default: + - kube-state-metrics:8080 + - name: leaderelection + type: bool + title: Leader Election + multi: false + required: true + show_user: true + default: true + - name: period + type: text + title: Period + multi: false + required: true + show_user: true + default: 10s + title: Kubernetes Deamonset metrics + description: Collect Kubernetes Deamonset metrics from kube_state_metrics diff --git a/test/packages/kubernetes/data_stream/state_daemonset/sample_event.json b/test/packages/kubernetes/data_stream/state_daemonset/sample_event.json new file mode 100644 index 0000000000..54b75c87c9 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_daemonset/sample_event.json @@ -0,0 +1,62 @@ +{ + "@timestamp": "2020-06-25T12:37:04.455Z", + "service": { + "address": "kube-state-metrics:8080", + "type": "kubernetes" + }, + "event": { + "module": "kubernetes", + "duration": 8648138, + "dataset": "kubernetes.daemonset" + }, + "ecs": { + "version": "1.5.0" + }, + "metricset": { + "name": "state_daemonset", + "period": 10000 + }, + "kubernetes": { + "daemonset": { + "name": "metricbeat", + "replicas": { + "available": 1, + "desired": 1, + "ready": 1, + "unavailable": 0 + } + }, + "labels": { + "k8s-app": "metricbeat" + }, + "namespace": "kube-system" + }, + "host": { + "mac": [ + "02:42:ac:11:00:0b" + ], + "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "architecture": "x86_64", + "os": { + "name": "CentOS Linux", + "kernel": "4.19.81", + "codename": "Core", + "platform": "centos", + "version": "7 (Core)", + "family": "redhat" + }, + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false, + "ip": [ + "172.17.0.11" + ] + }, + "agent": { + "version": "8.0.0", + "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", + "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "type": "metricbeat" + } +} \ No newline at end of file diff --git a/test/packages/kubernetes/data_stream/state_deployment/_dev/test/system/test-default-config.yml b/test/packages/kubernetes/data_stream/state_deployment/_dev/test/system/test-default-config.yml new file mode 100644 index 0000000000..0f4bd620c0 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_deployment/_dev/test/system/test-default-config.yml @@ -0,0 +1,6 @@ +service: kubernetes +data_stream: + vars: + hosts: + # this is the DNS name of the k8s service for kube-state-metrics deployment + - http://kube-state-metrics:8080 diff --git a/test/packages/kubernetes/data_stream/state_deployment/agent/stream/stream.yml.hbs b/test/packages/kubernetes/data_stream/state_deployment/agent/stream/stream.yml.hbs new file mode 100644 index 0000000000..205bc73767 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_deployment/agent/stream/stream.yml.hbs @@ -0,0 +1,10 @@ +metricsets: ["state_deployment"] +add_metadata: {{add_metadata}} +hosts: +{{#each hosts}} + - {{this}} +{{/each}} +period: {{period}} +{{#if leaderelection}} +condition: ${kubernetes_leaderelection.leader} == true +{{/if}} diff --git a/test/packages/kubernetes/data_stream/state_deployment/fields/agent.yml b/test/packages/kubernetes/data_stream/state_deployment/fields/agent.yml new file mode 100644 index 0000000000..da4e652c53 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_deployment/fields/agent.yml @@ -0,0 +1,198 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + diff --git a/test/packages/kubernetes/data_stream/state_deployment/fields/base-fields.yml b/test/packages/kubernetes/data_stream/state_deployment/fields/base-fields.yml new file mode 100644 index 0000000000..5ba440e428 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_deployment/fields/base-fields.yml @@ -0,0 +1,92 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. +- name: kubernetes + type: group + fields: + - name: pod.name + type: keyword + description: > + Kubernetes pod name + + - name: pod.uid + type: keyword + description: > + Kubernetes pod UID + + - name: pod.ip + type: ip + description: > + Kubernetes pod IP + + - name: namespace + type: keyword + description: > + Kubernetes namespace + + - name: node.name + type: keyword + description: > + Kubernetes node name + + - name: node.hostname + type: keyword + description: > + Kubernetes hostname as reported by the node’s kernel + + - name: labels.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes labels map + + - name: annotations.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes annotations map + + - name: selectors.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes Service selectors map + + - name: replicaset.name + dimension: true + type: keyword + description: > + Kubernetes replicaset name + + - name: deployment.name + dimension: true + type: keyword + description: > + Kubernetes deployment name + + - name: statefulset.name + type: keyword + description: > + Kubernetes statefulset name + + - name: container.name + type: keyword + description: > + Kubernetes container name + + - name: container.image + type: keyword + description: >- + Kubernetes container image diff --git a/test/packages/kubernetes/data_stream/state_deployment/fields/ecs.yml b/test/packages/kubernetes/data_stream/state_deployment/fields/ecs.yml new file mode 100644 index 0000000000..cd4e3a89b1 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_deployment/fields/ecs.yml @@ -0,0 +1,10 @@ +- external: ecs + name: ecs.version +- external: ecs + name: service.address +- external: ecs + name: service.type +- external: ecs + name: orchestrator.cluster.name +- external: ecs + name: orchestrator.cluster.url diff --git a/test/packages/kubernetes/data_stream/state_deployment/fields/fields.yml b/test/packages/kubernetes/data_stream/state_deployment/fields/fields.yml new file mode 100644 index 0000000000..51b6abb87e --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_deployment/fields/fields.yml @@ -0,0 +1,30 @@ +- name: kubernetes.deployment + type: group + fields: + - name: paused + type: boolean + description: | + Kubernetes deployment paused status + - name: replicas + type: group + fields: + - name: desired + type: integer + metric_type: gauge + description: | + Deployment number of desired replicas (spec) + - name: available + type: integer + metric_type: gauge + description: | + Deployment available replicas + - name: unavailable + type: integer + metric_type: gauge + description: | + Deployment unavailable replicas + - name: updated + type: integer + metric_type: gauge + description: | + Deployment updated replicas diff --git a/test/packages/kubernetes/data_stream/state_deployment/manifest.yml b/test/packages/kubernetes/data_stream/state_deployment/manifest.yml new file mode 100644 index 0000000000..29b7fece8f --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_deployment/manifest.yml @@ -0,0 +1,37 @@ +title: Kubernetes Deployment metrics +type: metrics +streams: + - input: kubernetes/metrics + enabled: false + vars: + - name: add_metadata + type: bool + title: Add Metadata + multi: false + required: true + show_user: true + default: true + - name: hosts + type: text + title: Hosts + multi: true + required: true + show_user: true + default: + - kube-state-metrics:8080 + - name: leaderelection + type: bool + title: Leader Election + multi: false + required: true + show_user: true + default: true + - name: period + type: text + title: Period + multi: false + required: true + show_user: true + default: 10s + title: Kubernetes Deployment metrics + description: Collect Kubernetes Deployment metrics from kube_state_metrics diff --git a/test/packages/kubernetes/data_stream/state_deployment/sample_event.json b/test/packages/kubernetes/data_stream/state_deployment/sample_event.json new file mode 100644 index 0000000000..fd79bacedd --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_deployment/sample_event.json @@ -0,0 +1,63 @@ +{ + "@timestamp": "2020-06-25T12:37:04.455Z", + "service": { + "address": "kube-state-metrics:8080", + "type": "kubernetes" + }, + "event": { + "module": "kubernetes", + "duration": 8648138, + "dataset": "kubernetes.deployment" + }, + "ecs": { + "version": "1.5.0" + }, + "metricset": { + "name": "state_deployment", + "period": 10000 + }, + "kubernetes": { + "deployment": { + "name": "metricbeat", + "replicas": { + "unavailable": 0, + "desired": 1, + "updated": 1, + "available": 1 + }, + "paused": false + }, + "labels": { + "k8s-app": "metricbeat" + }, + "namespace": "kube-system" + }, + "host": { + "mac": [ + "02:42:ac:11:00:0b" + ], + "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "architecture": "x86_64", + "os": { + "name": "CentOS Linux", + "kernel": "4.19.81", + "codename": "Core", + "platform": "centos", + "version": "7 (Core)", + "family": "redhat" + }, + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false, + "ip": [ + "172.17.0.11" + ] + }, + "agent": { + "version": "8.0.0", + "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", + "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "type": "metricbeat" + } +} \ No newline at end of file diff --git a/test/packages/kubernetes/data_stream/state_job/_dev/test/system/test-default-config.yml b/test/packages/kubernetes/data_stream/state_job/_dev/test/system/test-default-config.yml new file mode 100644 index 0000000000..0f4bd620c0 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_job/_dev/test/system/test-default-config.yml @@ -0,0 +1,6 @@ +service: kubernetes +data_stream: + vars: + hosts: + # this is the DNS name of the k8s service for kube-state-metrics deployment + - http://kube-state-metrics:8080 diff --git a/test/packages/kubernetes/data_stream/state_job/agent/stream/stream.yml.hbs b/test/packages/kubernetes/data_stream/state_job/agent/stream/stream.yml.hbs new file mode 100644 index 0000000000..b9903eff74 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_job/agent/stream/stream.yml.hbs @@ -0,0 +1,7 @@ +metricsets: ["state_job"] +add_metadata: {{add_metadata}} +hosts: +{{#each hosts}} + - {{this}} +{{/each}} +period: {{period}} diff --git a/test/packages/kubernetes/data_stream/state_job/fields/agent.yml b/test/packages/kubernetes/data_stream/state_job/fields/agent.yml new file mode 100644 index 0000000000..da4e652c53 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_job/fields/agent.yml @@ -0,0 +1,198 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + diff --git a/test/packages/kubernetes/data_stream/state_job/fields/base-fields.yml b/test/packages/kubernetes/data_stream/state_job/fields/base-fields.yml new file mode 100644 index 0000000000..d43ffd7964 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_job/fields/base-fields.yml @@ -0,0 +1,90 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. +- name: kubernetes + type: group + fields: + - name: pod.name + type: keyword + description: > + Kubernetes pod name + + - name: pod.uid + type: keyword + description: > + Kubernetes pod UID + + - name: pod.ip + type: ip + description: > + Kubernetes pod IP + + - name: namespace + type: keyword + description: > + Kubernetes namespace + + - name: node.name + type: keyword + description: > + Kubernetes node name + + - name: node.hostname + type: keyword + description: > + Kubernetes hostname as reported by the node’s kernel + + - name: labels.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes labels map + + - name: annotations.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes annotations map + + - name: selectors.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes Service selectors map + + - name: replicaset.name + type: keyword + description: > + Kubernetes replicaset name + + - name: deployment.name + type: keyword + description: > + Kubernetes deployment name + + - name: statefulset.name + type: keyword + description: > + Kubernetes statefulset name + + - name: container.name + type: keyword + description: > + Kubernetes container name + + - name: container.image + type: keyword + description: >- + Kubernetes container image diff --git a/test/packages/kubernetes/data_stream/state_job/fields/ecs.yml b/test/packages/kubernetes/data_stream/state_job/fields/ecs.yml new file mode 100644 index 0000000000..cd4e3a89b1 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_job/fields/ecs.yml @@ -0,0 +1,10 @@ +- external: ecs + name: ecs.version +- external: ecs + name: service.address +- external: ecs + name: service.type +- external: ecs + name: orchestrator.cluster.name +- external: ecs + name: orchestrator.cluster.url diff --git a/test/packages/kubernetes/data_stream/state_job/fields/fields.yml b/test/packages/kubernetes/data_stream/state_job/fields/fields.yml new file mode 100644 index 0000000000..dd96148fae --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_job/fields/fields.yml @@ -0,0 +1,76 @@ +- name: kubernetes.job + type: group + fields: + - name: name + dimension: true + type: keyword + description: > + The name of the job resource + + - name: pods + type: group + description: > + Pod metrics for the job + + fields: + - name: active + type: long + metric_type: gauge + description: Number of active pods + - name: failed + type: long + metric_type: gauge + description: Number of failed pods + - name: succeeded + type: long + metric_type: gauge + description: Number of successful pods + - name: time + type: group + description: Kubernetes job timestamps + fields: + - name: created + type: date + description: The time at which the job was created + - name: completed + type: date + description: The time at which the job completed + - name: completions + type: group + description: Kubernetes job completion settings + fields: + - name: desired + type: long + metric_type: gauge + description: The configured completion count for the job (Spec) + - name: parallelism + type: group + description: Kubernetes job parallelism settings + fields: + - name: desired + type: long + metric_type: gauge + description: The configured parallelism of the job (Spec) + - name: owner + type: group + description: Kubernetes job owner information + fields: + - name: name + type: keyword + description: The name of the resource that owns this job + - name: kind + type: keyword + description: The kind of resource that owns this job (eg. "CronJob") + - name: is_controller + type: keyword + description: Owner is controller ("true", "false", or `""`) + - name: status + type: group + description: Kubernetes job status information + fields: + - name: complete + type: keyword + description: Whether the job completed ("true", "false", or "unknown") + - name: failed + type: keyword + description: Whether the job failed ("true", "false", or "unknown") diff --git a/test/packages/kubernetes/data_stream/state_job/manifest.yml b/test/packages/kubernetes/data_stream/state_job/manifest.yml new file mode 100644 index 0000000000..5026046ff5 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_job/manifest.yml @@ -0,0 +1,30 @@ +title: Kubernetes Job metrics +type: metrics +streams: + - input: kubernetes/metrics + enabled: false + vars: + - name: add_metadata + type: bool + title: Add Metadata + multi: false + required: true + show_user: true + default: true + - name: hosts + type: text + title: Hosts + multi: true + required: true + show_user: true + default: + - kube-state-metrics:8080 + - name: period + type: text + title: Period + multi: false + required: true + show_user: true + default: 10s + title: Kubernetes Job metrics + description: Collect Kubernetes Job metrics from kube_state_metrics diff --git a/test/packages/kubernetes/data_stream/state_job/sample_event.json b/test/packages/kubernetes/data_stream/state_job/sample_event.json new file mode 100644 index 0000000000..c9ff59627c --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_job/sample_event.json @@ -0,0 +1,72 @@ +{ + "@timestamp": "2020-06-25T12:43:04.384Z", + "metricset": { + "name": "state_job", + "period": 10000 + }, + "service": { + "address": "kube-state-metrics:8080", + "type": "kubernetes" + }, + "ecs": { + "version": "1.5.0" + }, + "host": { + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false, + "ip": [ + "172.17.0.11" + ], + "mac": [ + "02:42:ac:11:00:0b" + ], + "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "architecture": "x86_64", + "os": { + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81", + "codename": "Core", + "platform": "centos", + "version": "7 (Core)" + } + }, + "event": { + "dataset": "kubernetes.job", + "module": "kubernetes", + "duration": 9482053 + }, + "kubernetes": { + "job": { + "completions": { + "desired": 1 + }, + "name": "sleep-30-ok-cron-27075645", + "owner": { + "is_controller": "true", + "kind": "CronJob", + "name": "sleep-30-ok-cron" + }, + "parallelism": { + "desired": 1 + }, + "pods": { + "active": 1, + "failed": 0, + "succeeded": 0 + }, + "time": { + "created": "2021-06-24T12:45:00.000Z" + } + }, + "namespace": "default" + }, + "agent": { + "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", + "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "type": "metricbeat", + "version": "8.0.0" + } +} \ No newline at end of file diff --git a/test/packages/kubernetes/data_stream/state_node/_dev/test/system/test-default-config.yml b/test/packages/kubernetes/data_stream/state_node/_dev/test/system/test-default-config.yml new file mode 100644 index 0000000000..0f4bd620c0 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_node/_dev/test/system/test-default-config.yml @@ -0,0 +1,6 @@ +service: kubernetes +data_stream: + vars: + hosts: + # this is the DNS name of the k8s service for kube-state-metrics deployment + - http://kube-state-metrics:8080 diff --git a/test/packages/kubernetes/data_stream/state_node/agent/stream/stream.yml.hbs b/test/packages/kubernetes/data_stream/state_node/agent/stream/stream.yml.hbs new file mode 100644 index 0000000000..1ddfb5e92f --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_node/agent/stream/stream.yml.hbs @@ -0,0 +1,10 @@ +metricsets: ["state_node"] +add_metadata: {{add_metadata}} +hosts: +{{#each hosts}} + - {{this}} +{{/each}} +period: {{period}} +{{#if leaderelection}} +condition: ${kubernetes_leaderelection.leader} == true +{{/if}} diff --git a/test/packages/kubernetes/data_stream/state_node/fields/agent.yml b/test/packages/kubernetes/data_stream/state_node/fields/agent.yml new file mode 100644 index 0000000000..da4e652c53 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_node/fields/agent.yml @@ -0,0 +1,198 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + diff --git a/test/packages/kubernetes/data_stream/state_node/fields/base-fields.yml b/test/packages/kubernetes/data_stream/state_node/fields/base-fields.yml new file mode 100644 index 0000000000..0e3072d1ef --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_node/fields/base-fields.yml @@ -0,0 +1,91 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. +- name: kubernetes + type: group + fields: + - name: pod.name + type: keyword + description: > + Kubernetes pod name + + - name: pod.uid + type: keyword + description: > + Kubernetes pod UID + + - name: pod.ip + type: ip + description: > + Kubernetes pod IP + + - name: namespace + type: keyword + description: > + Kubernetes namespace + + - name: node.name + dimension: true + type: keyword + description: > + Kubernetes node name + + - name: node.hostname + type: keyword + description: > + Kubernetes hostname as reported by the node’s kernel + + - name: labels.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes labels map + + - name: annotations.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes annotations map + + - name: selectors.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes Service selectors map + + - name: replicaset.name + type: keyword + description: > + Kubernetes replicaset name + + - name: deployment.name + type: keyword + description: > + Kubernetes deployment name + + - name: statefulset.name + type: keyword + description: > + Kubernetes statefulset name + + - name: container.name + type: keyword + description: > + Kubernetes container name + + - name: container.image + type: keyword + description: >- + Kubernetes container image diff --git a/test/packages/kubernetes/data_stream/state_node/fields/ecs.yml b/test/packages/kubernetes/data_stream/state_node/fields/ecs.yml new file mode 100644 index 0000000000..cd4e3a89b1 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_node/fields/ecs.yml @@ -0,0 +1,10 @@ +- external: ecs + name: ecs.version +- external: ecs + name: service.address +- external: ecs + name: service.type +- external: ecs + name: orchestrator.cluster.name +- external: ecs + name: orchestrator.cluster.url diff --git a/test/packages/kubernetes/data_stream/state_node/fields/fields.yml b/test/packages/kubernetes/data_stream/state_node/fields/fields.yml new file mode 100644 index 0000000000..c1eb215244 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_node/fields/fields.yml @@ -0,0 +1,69 @@ +- name: kubernetes.node + type: group + fields: + - name: status + type: group + fields: + - name: ready + type: keyword + description: | + Node ready status (true, false or unknown) + - name: unschedulable + type: boolean + description: | + Node unschedulable status + - name: disk_pressure + type: keyword + description: Node DiskPressure status (true, false or unknown) + - name: memory_pressure + type: keyword + description: Node MemoryPressure status (true, false or unknown) + - name: out_of_disk + type: keyword + description: Node OutOfDisk status (true, false or unknown) + - name: pid_pressure + type: keyword + description: Node PIDPressure status (true, false or unknown) + - name: cpu + type: group + fields: + - name: allocatable.cores + type: float + metric_type: gauge + description: | + Node CPU allocatable cores + - name: capacity.cores + type: long + metric_type: gauge + description: | + Node CPU capacity cores + - name: memory + type: group + fields: + - name: allocatable.bytes + type: long + format: bytes + unit: byte + metric_type: gauge + description: | + Node allocatable memory in bytes + - name: capacity.bytes + type: long + format: bytes + unit: byte + metric_type: gauge + description: | + Node memory capacity in bytes + - name: pod + type: group + fields: + - name: allocatable.total + type: long + metric_type: gauge + description: | + Node allocatable pods + - name: capacity.total + type: long + metric_type: gauge + description: | + Node pod capacity diff --git a/test/packages/kubernetes/data_stream/state_node/manifest.yml b/test/packages/kubernetes/data_stream/state_node/manifest.yml new file mode 100644 index 0000000000..294a71f51f --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_node/manifest.yml @@ -0,0 +1,37 @@ +title: Kubernetes Node metrics +type: metrics +streams: + - input: kubernetes/metrics + enabled: false + vars: + - name: add_metadata + type: bool + title: Add Metadata + multi: false + required: true + show_user: true + default: true + - name: hosts + type: text + title: Hosts + multi: true + required: true + show_user: true + default: + - kube-state-metrics:8080 + - name: leaderelection + type: bool + title: Leader Election + multi: false + required: true + show_user: true + default: true + - name: period + type: text + title: Period + multi: false + required: true + show_user: true + default: 10s + title: Kubernetes Node metrics + description: Collect Kubernetes Node metrics from kube_state_metrics diff --git a/test/packages/kubernetes/data_stream/state_node/sample_event.json b/test/packages/kubernetes/data_stream/state_node/sample_event.json new file mode 100644 index 0000000000..9b7f451ab6 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_node/sample_event.json @@ -0,0 +1,88 @@ +{ + "@timestamp": "2020-06-25T12:37:44.457Z", + "ecs": { + "version": "1.5.0" + }, + "host": { + "mac": [ + "02:42:ac:11:00:0b" + ], + "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "architecture": "x86_64", + "os": { + "kernel": "4.19.81", + "codename": "Core", + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux" + }, + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false, + "ip": [ + "172.17.0.11" + ] + }, + "metricset": { + "name": "state_node", + "period": 10000 + }, + "kubernetes": { + "node": { + "pod": { + "capacity": { + "total": 110 + }, + "allocatable": { + "total": 110 + } + }, + "memory": { + "capacity": { + "bytes": 16815325184 + }, + "allocatable": { + "bytes": 16815325184 + } + }, + "cpu": { + "allocatable": { + "cores": 4 + }, + "capacity": { + "cores": 4 + } + }, + "name": "minikube", + "status": { + "ready": "true", + "unschedulable": false + } + }, + "labels": { + "kubernetes_io/arch": "amd64", + "kubernetes_io/hostname": "minikube", + "kubernetes_io/os": "linux", + "node-role_kubernetes_io/master": "", + "beta_kubernetes_io/arch": "amd64", + "beta_kubernetes_io/os": "linux" + } + }, + "agent": { + "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", + "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "type": "metricbeat", + "version": "8.0.0" + }, + "service": { + "type": "kubernetes", + "address": "kube-state-metrics:8080" + }, + "event": { + "dataset": "kubernetes.node", + "module": "kubernetes", + "duration": 8194220 + } +} \ No newline at end of file diff --git a/test/packages/kubernetes/data_stream/state_persistentvolume/_dev/test/system/test-default-config.yml b/test/packages/kubernetes/data_stream/state_persistentvolume/_dev/test/system/test-default-config.yml new file mode 100644 index 0000000000..0f4bd620c0 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_persistentvolume/_dev/test/system/test-default-config.yml @@ -0,0 +1,6 @@ +service: kubernetes +data_stream: + vars: + hosts: + # this is the DNS name of the k8s service for kube-state-metrics deployment + - http://kube-state-metrics:8080 diff --git a/test/packages/kubernetes/data_stream/state_persistentvolume/agent/stream/stream.yml.hbs b/test/packages/kubernetes/data_stream/state_persistentvolume/agent/stream/stream.yml.hbs new file mode 100644 index 0000000000..b8556ce98b --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_persistentvolume/agent/stream/stream.yml.hbs @@ -0,0 +1,11 @@ +metricsets: ["state_persistentvolume"] +add_metadata: {{add_metadata}} +hosts: +{{#each hosts}} + - {{this}} +{{/each}} +period: {{period}} +{{#if leaderelection}} +condition: ${kubernetes_leaderelection.leader} == true +{{/if}} + diff --git a/test/packages/kubernetes/data_stream/state_persistentvolume/fields/agent.yml b/test/packages/kubernetes/data_stream/state_persistentvolume/fields/agent.yml new file mode 100644 index 0000000000..da4e652c53 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_persistentvolume/fields/agent.yml @@ -0,0 +1,198 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + diff --git a/test/packages/kubernetes/data_stream/state_persistentvolume/fields/base-fields.yml b/test/packages/kubernetes/data_stream/state_persistentvolume/fields/base-fields.yml new file mode 100644 index 0000000000..d43ffd7964 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_persistentvolume/fields/base-fields.yml @@ -0,0 +1,90 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. +- name: kubernetes + type: group + fields: + - name: pod.name + type: keyword + description: > + Kubernetes pod name + + - name: pod.uid + type: keyword + description: > + Kubernetes pod UID + + - name: pod.ip + type: ip + description: > + Kubernetes pod IP + + - name: namespace + type: keyword + description: > + Kubernetes namespace + + - name: node.name + type: keyword + description: > + Kubernetes node name + + - name: node.hostname + type: keyword + description: > + Kubernetes hostname as reported by the node’s kernel + + - name: labels.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes labels map + + - name: annotations.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes annotations map + + - name: selectors.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes Service selectors map + + - name: replicaset.name + type: keyword + description: > + Kubernetes replicaset name + + - name: deployment.name + type: keyword + description: > + Kubernetes deployment name + + - name: statefulset.name + type: keyword + description: > + Kubernetes statefulset name + + - name: container.name + type: keyword + description: > + Kubernetes container name + + - name: container.image + type: keyword + description: >- + Kubernetes container image diff --git a/test/packages/kubernetes/data_stream/state_persistentvolume/fields/ecs.yml b/test/packages/kubernetes/data_stream/state_persistentvolume/fields/ecs.yml new file mode 100644 index 0000000000..cd4e3a89b1 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_persistentvolume/fields/ecs.yml @@ -0,0 +1,10 @@ +- external: ecs + name: ecs.version +- external: ecs + name: service.address +- external: ecs + name: service.type +- external: ecs + name: orchestrator.cluster.name +- external: ecs + name: orchestrator.cluster.url diff --git a/test/packages/kubernetes/data_stream/state_persistentvolume/fields/fields.yml b/test/packages/kubernetes/data_stream/state_persistentvolume/fields/fields.yml new file mode 100644 index 0000000000..e441ac8f9c --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_persistentvolume/fields/fields.yml @@ -0,0 +1,18 @@ +- name: kubernetes.persistentvolume + type: group + fields: + - name: name + dimension: true + type: keyword + description: Volume name. + - name: capacity.bytes + type: long + unit: byte + metric_type: gauge + description: Volume capacity + - name: phase + type: keyword + description: Volume phase according to kubernetes + - name: storage_class + type: keyword + description: Storage class for the volume diff --git a/test/packages/kubernetes/data_stream/state_persistentvolume/manifest.yml b/test/packages/kubernetes/data_stream/state_persistentvolume/manifest.yml new file mode 100644 index 0000000000..1a6cbabc46 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_persistentvolume/manifest.yml @@ -0,0 +1,37 @@ +title: Kubernetes PersistentVolume metrics +type: metrics +streams: + - input: kubernetes/metrics + enabled: false + vars: + - name: add_metadata + type: bool + title: Add Metadata + multi: false + required: true + show_user: true + default: true + - name: hosts + type: text + title: Hosts + multi: true + required: true + show_user: true + default: + - kube-state-metrics:8080 + - name: leaderelection + type: bool + title: Leader Election + multi: false + required: true + show_user: true + default: true + - name: period + type: text + title: Period + multi: false + required: true + show_user: true + default: 10s + title: Kubernetes PersistentVolume metrics + description: Collect Kubernetes PersistentVolume metrics from kube_state_metrics diff --git a/test/packages/kubernetes/data_stream/state_persistentvolume/sample_event.json b/test/packages/kubernetes/data_stream/state_persistentvolume/sample_event.json new file mode 100644 index 0000000000..cbc6f7f846 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_persistentvolume/sample_event.json @@ -0,0 +1,60 @@ +{ + "@timestamp": "2020-06-25T12:43:54.412Z", + "ecs": { + "version": "1.5.0" + }, + "event": { + "module": "kubernetes", + "duration": 12149615, + "dataset": "kubernetes.persistentvolume" + }, + "agent": { + "version": "8.0.0", + "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", + "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "type": "metricbeat" + }, + "kubernetes": { + "persistentvolume": { + "capacity": { + "bytes": 10737418240 + }, + "phase": "Bound", + "storage_class": "manual", + "name": "task-pv-volume" + }, + "labels": { + "type": "local" + } + }, + "host": { + "ip": [ + "172.17.0.11" + ], + "mac": [ + "02:42:ac:11:00:0b" + ], + "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "architecture": "x86_64", + "os": { + "codename": "Core", + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81" + }, + "id": "b0e83d397c054b8a99a431072fe4617b", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "containerized": false + }, + "metricset": { + "period": 10000, + "name": "state_persistentvolume" + }, + "service": { + "address": "kube-state-metrics:8080", + "type": "kubernetes" + } +} \ No newline at end of file diff --git a/test/packages/kubernetes/data_stream/state_persistentvolumeclaim/_dev/test/system/test-default-config.yml b/test/packages/kubernetes/data_stream/state_persistentvolumeclaim/_dev/test/system/test-default-config.yml new file mode 100644 index 0000000000..0f4bd620c0 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_persistentvolumeclaim/_dev/test/system/test-default-config.yml @@ -0,0 +1,6 @@ +service: kubernetes +data_stream: + vars: + hosts: + # this is the DNS name of the k8s service for kube-state-metrics deployment + - http://kube-state-metrics:8080 diff --git a/test/packages/kubernetes/data_stream/state_persistentvolumeclaim/agent/stream/stream.yml.hbs b/test/packages/kubernetes/data_stream/state_persistentvolumeclaim/agent/stream/stream.yml.hbs new file mode 100644 index 0000000000..06ae236140 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_persistentvolumeclaim/agent/stream/stream.yml.hbs @@ -0,0 +1,10 @@ +metricsets: ["state_persistentvolumeclaim"] +add_metadata: {{add_metadata}} +hosts: +{{#each hosts}} + - {{this}} +{{/each}} +period: {{period}} +{{#if leaderelection}} +condition: ${kubernetes_leaderelection.leader} == true +{{/if}} diff --git a/test/packages/kubernetes/data_stream/state_persistentvolumeclaim/fields/agent.yml b/test/packages/kubernetes/data_stream/state_persistentvolumeclaim/fields/agent.yml new file mode 100644 index 0000000000..da4e652c53 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_persistentvolumeclaim/fields/agent.yml @@ -0,0 +1,198 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + diff --git a/test/packages/kubernetes/data_stream/state_persistentvolumeclaim/fields/base-fields.yml b/test/packages/kubernetes/data_stream/state_persistentvolumeclaim/fields/base-fields.yml new file mode 100644 index 0000000000..d43ffd7964 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_persistentvolumeclaim/fields/base-fields.yml @@ -0,0 +1,90 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. +- name: kubernetes + type: group + fields: + - name: pod.name + type: keyword + description: > + Kubernetes pod name + + - name: pod.uid + type: keyword + description: > + Kubernetes pod UID + + - name: pod.ip + type: ip + description: > + Kubernetes pod IP + + - name: namespace + type: keyword + description: > + Kubernetes namespace + + - name: node.name + type: keyword + description: > + Kubernetes node name + + - name: node.hostname + type: keyword + description: > + Kubernetes hostname as reported by the node’s kernel + + - name: labels.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes labels map + + - name: annotations.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes annotations map + + - name: selectors.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes Service selectors map + + - name: replicaset.name + type: keyword + description: > + Kubernetes replicaset name + + - name: deployment.name + type: keyword + description: > + Kubernetes deployment name + + - name: statefulset.name + type: keyword + description: > + Kubernetes statefulset name + + - name: container.name + type: keyword + description: > + Kubernetes container name + + - name: container.image + type: keyword + description: >- + Kubernetes container image diff --git a/test/packages/kubernetes/data_stream/state_persistentvolumeclaim/fields/ecs.yml b/test/packages/kubernetes/data_stream/state_persistentvolumeclaim/fields/ecs.yml new file mode 100644 index 0000000000..cd4e3a89b1 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_persistentvolumeclaim/fields/ecs.yml @@ -0,0 +1,10 @@ +- external: ecs + name: ecs.version +- external: ecs + name: service.address +- external: ecs + name: service.type +- external: ecs + name: orchestrator.cluster.name +- external: ecs + name: orchestrator.cluster.url diff --git a/test/packages/kubernetes/data_stream/state_persistentvolumeclaim/fields/fields.yml b/test/packages/kubernetes/data_stream/state_persistentvolumeclaim/fields/fields.yml new file mode 100644 index 0000000000..6f11ce66b7 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_persistentvolumeclaim/fields/fields.yml @@ -0,0 +1,24 @@ +- name: kubernetes.persistentvolumeclaim + type: group + fields: + - name: name + dimension: true + type: keyword + description: PVC name. + - name: volume_name + type: keyword + description: Binded volume name. + - name: request_storage.bytes + type: long + unit: byte + metric_type: gauge + description: Requested capacity. + - name: phase + type: keyword + description: PVC phase. + - name: access_mode + type: keyword + description: Access mode. + - name: storage_class + type: keyword + description: Storage class for the PVC. diff --git a/test/packages/kubernetes/data_stream/state_persistentvolumeclaim/manifest.yml b/test/packages/kubernetes/data_stream/state_persistentvolumeclaim/manifest.yml new file mode 100644 index 0000000000..a825fda1c1 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_persistentvolumeclaim/manifest.yml @@ -0,0 +1,37 @@ +title: Kubernetes PersistentVolumeClaim metrics +type: metrics +streams: + - input: kubernetes/metrics + enabled: false + vars: + - name: add_metadata + type: bool + title: Add Metadata + multi: false + required: true + show_user: true + default: true + - name: hosts + type: text + title: Hosts + multi: true + required: true + show_user: true + default: + - kube-state-metrics:8080 + - name: leaderelection + type: bool + title: Leader Election + multi: false + required: true + show_user: true + default: true + - name: period + type: text + title: Period + multi: false + required: true + show_user: true + default: 10s + title: Kubernetes PersistentVolumeClaim metrics + description: Collect Kubernetes PersistentVolumeClaim metrics from kube_state_metrics diff --git a/test/packages/kubernetes/data_stream/state_persistentvolumeclaim/sample_event.json b/test/packages/kubernetes/data_stream/state_persistentvolumeclaim/sample_event.json new file mode 100644 index 0000000000..0a1204964a --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_persistentvolumeclaim/sample_event.json @@ -0,0 +1,60 @@ +{ + "@timestamp": "2020-06-25T12:44:44.418Z", + "event": { + "dataset": "kubernetes.persistentvolumeclaim", + "module": "kubernetes", + "duration": 5698588 + }, + "metricset": { + "name": "state_persistentvolumeclaim", + "period": 10000 + }, + "service": { + "address": "kube-state-metrics:8080", + "type": "kubernetes" + }, + "kubernetes": { + "namespace": "default", + "persistentvolumeclaim": { + "phase": "Bound", + "storage_class": "manual", + "volume_name": "task-pv-volume", + "name": "task-pv-claim", + "request_storage": { + "bytes": 3221225472 + }, + "access_mode": "ReadWriteOnce" + } + }, + "agent": { + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", + "id": "a6147a6e-6626-4a84-9907-f372f6c61eee" + }, + "ecs": { + "version": "1.5.0" + }, + "host": { + "os": { + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81", + "codename": "Core" + }, + "id": "b0e83d397c054b8a99a431072fe4617b", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "containerized": false, + "ip": [ + "172.17.0.11" + ], + "mac": [ + "02:42:ac:11:00:0b" + ], + "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "architecture": "x86_64" + } +} \ No newline at end of file diff --git a/test/packages/kubernetes/data_stream/state_pod/fields/base-fields.yml b/test/packages/kubernetes/data_stream/state_pod/fields/base-fields.yml index bd2abd6f22..31b09fa299 100644 --- a/test/packages/kubernetes/data_stream/state_pod/fields/base-fields.yml +++ b/test/packages/kubernetes/data_stream/state_pod/fields/base-fields.yml @@ -14,11 +14,13 @@ type: group fields: - name: pod.name + dimension: true type: keyword description: > Kubernetes pod name - name: pod.uid + dimension: true type: keyword description: > Kubernetes pod UID diff --git a/test/packages/kubernetes/data_stream/state_pod/fields/ecs.yml b/test/packages/kubernetes/data_stream/state_pod/fields/ecs.yml index e60b6629bb..31cb2817b8 100644 --- a/test/packages/kubernetes/data_stream/state_pod/fields/ecs.yml +++ b/test/packages/kubernetes/data_stream/state_pod/fields/ecs.yml @@ -1,16 +1,12 @@ -- name: ecs.version - type: keyword - description: ECS version -- name: service.address - type: keyword - description: Service address -- name: service.type - type: keyword - description: Service type -- name: container.runtime - type: keyword - description: Runtime managing this container -- name: orchestrator.cluster.name - external: ecs -- name: orchestrator.cluster.url - external: ecs +- external: ecs + name: ecs.version +- external: ecs + name: service.address +- external: ecs + name: service.type +- external: ecs + name: container.runtime +- external: ecs + name: orchestrator.cluster.name +- external: ecs + name: orchestrator.cluster.url diff --git a/test/packages/kubernetes/data_stream/state_pod/manifest.yml b/test/packages/kubernetes/data_stream/state_pod/manifest.yml index f000ea29cd..1651226365 100644 --- a/test/packages/kubernetes/data_stream/state_pod/manifest.yml +++ b/test/packages/kubernetes/data_stream/state_pod/manifest.yml @@ -1,5 +1,4 @@ title: Kubernetes Pod metrics -release: experimental type: metrics streams: - input: kubernetes/metrics diff --git a/test/packages/kubernetes/data_stream/state_replicaset/_dev/test/system/test-default-config.yml b/test/packages/kubernetes/data_stream/state_replicaset/_dev/test/system/test-default-config.yml new file mode 100644 index 0000000000..0f4bd620c0 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_replicaset/_dev/test/system/test-default-config.yml @@ -0,0 +1,6 @@ +service: kubernetes +data_stream: + vars: + hosts: + # this is the DNS name of the k8s service for kube-state-metrics deployment + - http://kube-state-metrics:8080 diff --git a/test/packages/kubernetes/data_stream/state_replicaset/agent/stream/stream.yml.hbs b/test/packages/kubernetes/data_stream/state_replicaset/agent/stream/stream.yml.hbs new file mode 100644 index 0000000000..8b4cdd5828 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_replicaset/agent/stream/stream.yml.hbs @@ -0,0 +1,10 @@ +metricsets: ["state_replicaset"] +add_metadata: {{add_metadata}} +hosts: +{{#each hosts}} + - {{this}} +{{/each}} +period: {{period}} +{{#if leaderelection}} +condition: ${kubernetes_leaderelection.leader} == true +{{/if}} diff --git a/test/packages/kubernetes/data_stream/state_replicaset/fields/agent.yml b/test/packages/kubernetes/data_stream/state_replicaset/fields/agent.yml new file mode 100644 index 0000000000..da4e652c53 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_replicaset/fields/agent.yml @@ -0,0 +1,198 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + diff --git a/test/packages/kubernetes/data_stream/state_replicaset/fields/base-fields.yml b/test/packages/kubernetes/data_stream/state_replicaset/fields/base-fields.yml new file mode 100644 index 0000000000..9db6054809 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_replicaset/fields/base-fields.yml @@ -0,0 +1,92 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. +- name: kubernetes + type: group + fields: + - name: pod.name + type: keyword + description: > + Kubernetes pod name + + - name: pod.uid + type: keyword + description: > + Kubernetes pod UID + + - name: pod.ip + type: ip + description: > + Kubernetes pod IP + + - name: namespace + type: keyword + description: > + Kubernetes namespace + + - name: node.name + type: keyword + description: > + Kubernetes node name + + - name: node.hostname + type: keyword + description: > + Kubernetes hostname as reported by the node’s kernel + + - name: labels.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes labels map + + - name: annotations.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes annotations map + + - name: selectors.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes Service selectors map + + - name: replicaset.name + dimensiont: true + type: keyword + description: > + Kubernetes replicaset name + + - name: deployment.name + dimensiont: true + type: keyword + description: > + Kubernetes deployment name + + - name: statefulset.name + type: keyword + description: > + Kubernetes statefulset name + + - name: container.name + type: keyword + description: > + Kubernetes container name + + - name: container.image + type: keyword + description: >- + Kubernetes container image diff --git a/test/packages/kubernetes/data_stream/state_replicaset/fields/ecs.yml b/test/packages/kubernetes/data_stream/state_replicaset/fields/ecs.yml new file mode 100644 index 0000000000..cd4e3a89b1 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_replicaset/fields/ecs.yml @@ -0,0 +1,10 @@ +- external: ecs + name: ecs.version +- external: ecs + name: service.address +- external: ecs + name: service.type +- external: ecs + name: orchestrator.cluster.name +- external: ecs + name: orchestrator.cluster.url diff --git a/test/packages/kubernetes/data_stream/state_replicaset/fields/fields.yml b/test/packages/kubernetes/data_stream/state_replicaset/fields/fields.yml new file mode 100644 index 0000000000..40928a7713 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_replicaset/fields/fields.yml @@ -0,0 +1,31 @@ +- name: kubernetes.replicaset + type: group + fields: + - name: replicas + type: group + fields: + - name: available + type: long + metric_type: gauge + description: | + The number of replicas per ReplicaSet + - name: desired + type: long + metric_type: gauge + description: | + The number of replicas per ReplicaSet + - name: ready + type: long + metric_type: gauge + description: | + The number of ready replicas per ReplicaSet + - name: observed + type: long + metric_type: gauge + description: | + The generation observed by the ReplicaSet controller + - name: labeled + type: long + metric_type: gauge + description: | + The number of fully labeled replicas per ReplicaSet diff --git a/test/packages/kubernetes/data_stream/state_replicaset/manifest.yml b/test/packages/kubernetes/data_stream/state_replicaset/manifest.yml new file mode 100644 index 0000000000..fd6847b613 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_replicaset/manifest.yml @@ -0,0 +1,37 @@ +title: Kubernetes state_replicaset metrics +type: metrics +streams: + - input: kubernetes/metrics + enabled: false + vars: + - name: add_metadata + type: bool + title: Add Metadata + multi: false + required: true + show_user: true + default: true + - name: hosts + type: text + title: Hosts + multi: true + required: true + show_user: true + default: + - kube-state-metrics:8080 + - name: leaderelection + type: bool + title: Leader Election + multi: false + required: true + show_user: true + default: true + - name: period + type: text + title: Period + multi: false + required: true + show_user: true + default: 10s + title: Kubernetes Replicaset metrics + description: Collect Kubernetes Replicaset metrics from kube_state_metrics diff --git a/test/packages/kubernetes/data_stream/state_replicaset/sample_event.json b/test/packages/kubernetes/data_stream/state_replicaset/sample_event.json new file mode 100644 index 0000000000..e5506863d7 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_replicaset/sample_event.json @@ -0,0 +1,69 @@ +{ + "@timestamp": "2020-06-25T12:38:54.482Z", + "service": { + "address": "kube-state-metrics:8080", + "type": "kubernetes" + }, + "metricset": { + "period": 10000, + "name": "state_replicaset" + }, + "event": { + "module": "kubernetes", + "duration": 5456128, + "dataset": "kubernetes.replicaset" + }, + "kubernetes": { + "namespace": "kube-system", + "replicaset": { + "name": "nginx-ingress-controller-6fc5bcc8c9", + "replicas": { + "labeled": 1, + "ready": 1, + "available": 1, + "observed": 1, + "desired": 1 + } + }, + "deployment": { + "name": "nginx-ingress-controller" + }, + "labels": { + "app_kubernetes_io/part-of": "kube-system", + "pod-template-hash": "6fc5bcc8c9", + "addonmanager_kubernetes_io/mode": "Reconcile", + "app_kubernetes_io/name": "nginx-ingress-controller" + } + }, + "agent": { + "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", + "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "type": "metricbeat", + "version": "8.0.0" + }, + "ecs": { + "version": "1.5.0" + }, + "host": { + "containerized": false, + "ip": [ + "172.17.0.11" + ], + "mac": [ + "02:42:ac:11:00:0b" + ], + "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "architecture": "x86_64", + "os": { + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81", + "codename": "Core" + }, + "id": "b0e83d397c054b8a99a431072fe4617b" + } +} \ No newline at end of file diff --git a/test/packages/kubernetes/data_stream/state_resourcequota/_dev/test/system/test-default-config.yml b/test/packages/kubernetes/data_stream/state_resourcequota/_dev/test/system/test-default-config.yml new file mode 100644 index 0000000000..0f4bd620c0 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_resourcequota/_dev/test/system/test-default-config.yml @@ -0,0 +1,6 @@ +service: kubernetes +data_stream: + vars: + hosts: + # this is the DNS name of the k8s service for kube-state-metrics deployment + - http://kube-state-metrics:8080 diff --git a/test/packages/kubernetes/data_stream/state_resourcequota/agent/stream/stream.yml.hbs b/test/packages/kubernetes/data_stream/state_resourcequota/agent/stream/stream.yml.hbs new file mode 100644 index 0000000000..2b7da2e386 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_resourcequota/agent/stream/stream.yml.hbs @@ -0,0 +1,10 @@ +metricsets: ["state_resourcequota"] +add_metadata: {{add_metadata}} +hosts: +{{#each hosts}} + - {{this}} +{{/each}} +period: {{period}} +{{#if leaderelection}} +condition: ${kubernetes_leaderelection.leader} == true +{{/if}} diff --git a/test/packages/kubernetes/data_stream/state_resourcequota/fields/agent.yml b/test/packages/kubernetes/data_stream/state_resourcequota/fields/agent.yml new file mode 100644 index 0000000000..da4e652c53 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_resourcequota/fields/agent.yml @@ -0,0 +1,198 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + diff --git a/test/packages/kubernetes/data_stream/state_resourcequota/fields/base-fields.yml b/test/packages/kubernetes/data_stream/state_resourcequota/fields/base-fields.yml new file mode 100644 index 0000000000..d43ffd7964 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_resourcequota/fields/base-fields.yml @@ -0,0 +1,90 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. +- name: kubernetes + type: group + fields: + - name: pod.name + type: keyword + description: > + Kubernetes pod name + + - name: pod.uid + type: keyword + description: > + Kubernetes pod UID + + - name: pod.ip + type: ip + description: > + Kubernetes pod IP + + - name: namespace + type: keyword + description: > + Kubernetes namespace + + - name: node.name + type: keyword + description: > + Kubernetes node name + + - name: node.hostname + type: keyword + description: > + Kubernetes hostname as reported by the node’s kernel + + - name: labels.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes labels map + + - name: annotations.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes annotations map + + - name: selectors.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes Service selectors map + + - name: replicaset.name + type: keyword + description: > + Kubernetes replicaset name + + - name: deployment.name + type: keyword + description: > + Kubernetes deployment name + + - name: statefulset.name + type: keyword + description: > + Kubernetes statefulset name + + - name: container.name + type: keyword + description: > + Kubernetes container name + + - name: container.image + type: keyword + description: >- + Kubernetes container image diff --git a/test/packages/kubernetes/data_stream/state_resourcequota/fields/ecs.yml b/test/packages/kubernetes/data_stream/state_resourcequota/fields/ecs.yml new file mode 100644 index 0000000000..cd4e3a89b1 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_resourcequota/fields/ecs.yml @@ -0,0 +1,10 @@ +- external: ecs + name: ecs.version +- external: ecs + name: service.address +- external: ecs + name: service.type +- external: ecs + name: orchestrator.cluster.name +- external: ecs + name: orchestrator.cluster.url diff --git a/test/packages/kubernetes/data_stream/state_resourcequota/fields/fields.yml b/test/packages/kubernetes/data_stream/state_resourcequota/fields/fields.yml new file mode 100644 index 0000000000..530619270c --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_resourcequota/fields/fields.yml @@ -0,0 +1,24 @@ +- name: kubernetes.resourcequota + type: group + fields: + - name: created.sec + type: double + unit: s + metric_type: gauge + description: Epoch seconds since the ResourceQuota was created + - name: quota + type: double + metric_type: gauge + description: Quota informed (hard or used) for the resource + - name: name + dimension: true + type: keyword + description: ResourceQuota name + - name: type + dimension: true + type: keyword + description: Quota information type, `hard` or `used` + - name: resource + dimension: true + type: keyword + description: Resource name the quota applies to diff --git a/test/packages/kubernetes/data_stream/state_resourcequota/manifest.yml b/test/packages/kubernetes/data_stream/state_resourcequota/manifest.yml new file mode 100644 index 0000000000..c06b079d74 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_resourcequota/manifest.yml @@ -0,0 +1,37 @@ +title: Kubernetes ResourceQuota metrics +type: metrics +streams: + - input: kubernetes/metrics + enabled: false + vars: + - name: add_metadata + type: bool + title: Add Metadata + multi: false + required: true + show_user: true + default: true + - name: hosts + type: text + title: Hosts + multi: true + required: true + show_user: true + default: + - kube-state-metrics:8080 + - name: leaderelection + type: bool + title: Leader Election + multi: false + required: true + show_user: true + default: true + - name: period + type: text + title: Period + multi: false + required: true + show_user: true + default: 10s + title: Kubernetes ResourceQuota metrics + description: Collect Kubernetes ResourceQuota metrics from kube_state_metrics diff --git a/test/packages/kubernetes/data_stream/state_resourcequota/sample_event.json b/test/packages/kubernetes/data_stream/state_resourcequota/sample_event.json new file mode 100644 index 0000000000..d3603943f0 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_resourcequota/sample_event.json @@ -0,0 +1,56 @@ +{ + "@timestamp": "2020-06-25T12:45:04.416Z", + "metricset": { + "name": "state_resourcequota", + "period": 10000 + }, + "host": { + "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "architecture": "x86_64", + "os": { + "codename": "Core", + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81" + }, + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false, + "ip": [ + "172.17.0.11" + ], + "mac": [ + "02:42:ac:11:00:0b" + ] + }, + "service": { + "address": "kube-state-metrics:8080", + "type": "kubernetes" + }, + "event": { + "dataset": "kubernetes.resourcequota", + "module": "kubernetes", + "duration": 6324269 + }, + "agent": { + "id": "a6147a6e-6626-4a84-9907-f372f6c61eee", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487" + }, + "ecs": { + "version": "1.5.0" + }, + "kubernetes": { + "namespace": "quota-object-example", + "resourcequota": { + "name": "object-quota-demo", + "resource": "persistentvolumeclaims", + "type": "hard", + "quota": 1 + } + } +} \ No newline at end of file diff --git a/test/packages/kubernetes/data_stream/state_service/agent/stream/stream.yml.hbs b/test/packages/kubernetes/data_stream/state_service/agent/stream/stream.yml.hbs new file mode 100644 index 0000000000..1343496786 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_service/agent/stream/stream.yml.hbs @@ -0,0 +1,10 @@ +metricsets: ["state_service"] +add_metadata: {{add_metadata}} +hosts: +{{#each hosts}} + - {{this}} +{{/each}} +period: {{period}} +{{#if leaderelection}} +condition: ${kubernetes_leaderelection.leader} == true +{{/if}} diff --git a/test/packages/kubernetes/data_stream/state_service/fields/agent.yml b/test/packages/kubernetes/data_stream/state_service/fields/agent.yml new file mode 100644 index 0000000000..da4e652c53 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_service/fields/agent.yml @@ -0,0 +1,198 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + diff --git a/test/packages/kubernetes/data_stream/state_service/fields/base-fields.yml b/test/packages/kubernetes/data_stream/state_service/fields/base-fields.yml new file mode 100644 index 0000000000..d43ffd7964 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_service/fields/base-fields.yml @@ -0,0 +1,90 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. +- name: kubernetes + type: group + fields: + - name: pod.name + type: keyword + description: > + Kubernetes pod name + + - name: pod.uid + type: keyword + description: > + Kubernetes pod UID + + - name: pod.ip + type: ip + description: > + Kubernetes pod IP + + - name: namespace + type: keyword + description: > + Kubernetes namespace + + - name: node.name + type: keyword + description: > + Kubernetes node name + + - name: node.hostname + type: keyword + description: > + Kubernetes hostname as reported by the node’s kernel + + - name: labels.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes labels map + + - name: annotations.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes annotations map + + - name: selectors.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes Service selectors map + + - name: replicaset.name + type: keyword + description: > + Kubernetes replicaset name + + - name: deployment.name + type: keyword + description: > + Kubernetes deployment name + + - name: statefulset.name + type: keyword + description: > + Kubernetes statefulset name + + - name: container.name + type: keyword + description: > + Kubernetes container name + + - name: container.image + type: keyword + description: >- + Kubernetes container image diff --git a/test/packages/kubernetes/data_stream/state_service/fields/ecs.yml b/test/packages/kubernetes/data_stream/state_service/fields/ecs.yml new file mode 100644 index 0000000000..cd4e3a89b1 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_service/fields/ecs.yml @@ -0,0 +1,10 @@ +- external: ecs + name: ecs.version +- external: ecs + name: service.address +- external: ecs + name: service.type +- external: ecs + name: orchestrator.cluster.name +- external: ecs + name: orchestrator.cluster.url diff --git a/test/packages/kubernetes/data_stream/state_service/fields/fields.yml b/test/packages/kubernetes/data_stream/state_service/fields/fields.yml new file mode 100644 index 0000000000..0bec402860 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_service/fields/fields.yml @@ -0,0 +1,31 @@ +- name: kubernetes.service + type: group + fields: + - name: name + dimension: true + type: keyword + description: Service name. + - name: cluster_ip + type: keyword + description: Internal IP for the service. + - name: external_name + type: keyword + description: Service external DNS name + - name: external_ip + type: keyword + description: Service external IP + - name: load_balancer_ip + type: keyword + description: Load Balancer service IP + - name: type + type: keyword + description: Service type + - name: ingress_ip + type: keyword + description: Ingress IP + - name: ingress_hostname + type: keyword + description: Ingress Hostname + - name: created + type: date + description: Service creation date diff --git a/test/packages/kubernetes/data_stream/state_service/manifest.yml b/test/packages/kubernetes/data_stream/state_service/manifest.yml new file mode 100644 index 0000000000..9c1214316d --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_service/manifest.yml @@ -0,0 +1,37 @@ +title: Kubernetes Service metrics +type: metrics +streams: + - input: kubernetes/metrics + enabled: false + vars: + - name: add_metadata + type: bool + title: Add Metadata + multi: false + required: true + show_user: true + default: true + - name: hosts + type: text + title: Hosts + multi: true + required: true + show_user: true + default: + - kube-state-metrics:8080 + - name: leaderelection + type: bool + title: Leader Election + multi: false + required: true + show_user: true + default: true + - name: period + type: text + title: Period + multi: false + required: true + show_user: true + default: 10s + title: Kubernetes Service metrics + description: Collect Kubernetes Service metrics from kube_state_metrics diff --git a/test/packages/kubernetes/data_stream/state_service/sample_event.json b/test/packages/kubernetes/data_stream/state_service/sample_event.json new file mode 100644 index 0000000000..a6cd05ed17 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_service/sample_event.json @@ -0,0 +1,62 @@ +{ + "@timestamp": "2020-06-25T12:39:24.389Z", + "kubernetes": { + "labels": { + "kubernetes_io_minikube_addons_endpoint": "metrics-server", + "kubernetes_io_name": "Metrics-server", + "addonmanager_kubernetes_io_mode": "Reconcile", + "kubernetes_io_minikube_addons": "metrics-server" + }, + "service": { + "name": "metrics-server", + "created": "2020-06-10T09:02:27.000Z", + "cluster_ip": "10.96.124.248", + "type": "ClusterIP" + }, + "namespace": "kube-system" + }, + "event": { + "dataset": "kubernetes.service", + "module": "kubernetes", + "duration": 10966648 + }, + "metricset": { + "name": "state_service", + "period": 10000 + }, + "host": { + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false, + "ip": [ + "172.17.0.11" + ], + "mac": [ + "02:42:ac:11:00:0b" + ], + "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "architecture": "x86_64", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "os": { + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81", + "codename": "Core" + } + }, + "agent": { + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", + "id": "a6147a6e-6626-4a84-9907-f372f6c61eee" + }, + "ecs": { + "version": "1.5.0" + }, + "service": { + "address": "kube-state-metrics:8080", + "type": "kubernetes" + } +} \ No newline at end of file diff --git a/test/packages/kubernetes/data_stream/state_statefulset/_dev/test/system/test-default-config.yml b/test/packages/kubernetes/data_stream/state_statefulset/_dev/test/system/test-default-config.yml new file mode 100644 index 0000000000..0f4bd620c0 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_statefulset/_dev/test/system/test-default-config.yml @@ -0,0 +1,6 @@ +service: kubernetes +data_stream: + vars: + hosts: + # this is the DNS name of the k8s service for kube-state-metrics deployment + - http://kube-state-metrics:8080 diff --git a/test/packages/kubernetes/data_stream/state_statefulset/agent/stream/stream.yml.hbs b/test/packages/kubernetes/data_stream/state_statefulset/agent/stream/stream.yml.hbs new file mode 100644 index 0000000000..2ecf8cc5f3 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_statefulset/agent/stream/stream.yml.hbs @@ -0,0 +1,10 @@ +metricsets: ["state_statefulset"] +add_metadata: {{add_metadata}} +hosts: +{{#each hosts}} + - {{this}} +{{/each}} +period: {{period}} +{{#if leaderelection}} +condition: ${kubernetes_leaderelection.leader} == true +{{/if}} diff --git a/test/packages/kubernetes/data_stream/state_statefulset/fields/agent.yml b/test/packages/kubernetes/data_stream/state_statefulset/fields/agent.yml new file mode 100644 index 0000000000..da4e652c53 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_statefulset/fields/agent.yml @@ -0,0 +1,198 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + diff --git a/test/packages/kubernetes/data_stream/state_statefulset/fields/base-fields.yml b/test/packages/kubernetes/data_stream/state_statefulset/fields/base-fields.yml new file mode 100644 index 0000000000..b6151d62fb --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_statefulset/fields/base-fields.yml @@ -0,0 +1,91 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. +- name: kubernetes + type: group + fields: + - name: pod.name + type: keyword + description: > + Kubernetes pod name + + - name: pod.uid + type: keyword + description: > + Kubernetes pod UID + + - name: pod.ip + type: ip + description: > + Kubernetes pod IP + + - name: namespace + type: keyword + description: > + Kubernetes namespace + + - name: node.name + type: keyword + description: > + Kubernetes node name + + - name: node.hostname + type: keyword + description: > + Kubernetes hostname as reported by the node’s kernel + + - name: labels.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes labels map + + - name: annotations.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes annotations map + + - name: selectors.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes Service selectors map + + - name: replicaset.name + type: keyword + description: > + Kubernetes replicaset name + + - name: deployment.name + type: keyword + description: > + Kubernetes deployment name + + - name: statefulset.name + dimensions: true + type: keyword + description: > + Kubernetes statefulset name + + - name: container.name + type: keyword + description: > + Kubernetes container name + + - name: container.image + type: keyword + description: >- + Kubernetes container image diff --git a/test/packages/kubernetes/data_stream/state_statefulset/fields/ecs.yml b/test/packages/kubernetes/data_stream/state_statefulset/fields/ecs.yml new file mode 100644 index 0000000000..cd4e3a89b1 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_statefulset/fields/ecs.yml @@ -0,0 +1,10 @@ +- external: ecs + name: ecs.version +- external: ecs + name: service.address +- external: ecs + name: service.type +- external: ecs + name: orchestrator.cluster.name +- external: ecs + name: orchestrator.cluster.url diff --git a/test/packages/kubernetes/data_stream/state_statefulset/fields/fields.yml b/test/packages/kubernetes/data_stream/state_statefulset/fields/fields.yml new file mode 100644 index 0000000000..e28adddcaa --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_statefulset/fields/fields.yml @@ -0,0 +1,40 @@ +- name: kubernetes.statefulset + type: group + fields: + - name: created + type: long + metric_type: gauge + description: | + The creation timestamp (epoch) for StatefulSet + - name: replicas + type: group + fields: + - name: observed + type: long + metric_type: gauge + description: | + The number of observed replicas per StatefulSet + - name: desired + type: long + metric_type: gauge + description: | + The number of desired replicas per StatefulSet + - name: ready + type: long + metric_type: gauge + description: > + The number of ready replicas per StatefulSet + + - name: generation + type: group + fields: + - name: observed + type: long + metric_type: gauge + description: | + The observed generation per StatefulSet + - name: desired + type: long + metric_type: gauge + description: | + The desired generation per StatefulSet diff --git a/test/packages/kubernetes/data_stream/state_statefulset/manifest.yml b/test/packages/kubernetes/data_stream/state_statefulset/manifest.yml new file mode 100644 index 0000000000..ea6c382220 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_statefulset/manifest.yml @@ -0,0 +1,37 @@ +title: Kubernetes StatefulSet metrics +type: metrics +streams: + - input: kubernetes/metrics + enabled: false + vars: + - name: add_metadata + type: bool + title: Add Metadata + multi: false + required: true + show_user: true + default: true + - name: hosts + type: text + title: Hosts + multi: true + required: true + show_user: true + default: + - kube-state-metrics:8080 + - name: leaderelection + type: bool + title: Leader Election + multi: false + required: true + show_user: true + default: true + - name: period + type: text + title: Period + multi: false + required: true + show_user: true + default: 10s + title: Kubernetes StatefulSet metrics + description: Collect Kubernetes StatefulSet metrics from kube_state_metrics diff --git a/test/packages/kubernetes/data_stream/state_statefulset/sample_event.json b/test/packages/kubernetes/data_stream/state_statefulset/sample_event.json new file mode 100644 index 0000000000..9a87bc3df2 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_statefulset/sample_event.json @@ -0,0 +1,62 @@ +{ + "@timestamp": "2020-06-25T12:39:24.389Z", + "kubernetes": { + "namespace": "default", + "statefulset": { + "created": 1511989697, + "generation": { + "desired": 4, + "observed": 2 + }, + "name": "mysql", + "replicas": { + "desired": 5, + "observed": 2 + } + } + }, + "event": { + "dataset": "kubernetes.statefulset", + "module": "kubernetes", + "duration": 10966648 + }, + "metricset": { + "name": "state_statefulset", + "period": 10000 + }, + "host": { + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false, + "ip": [ + "172.17.0.11" + ], + "mac": [ + "02:42:ac:11:00:0b" + ], + "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "architecture": "x86_64", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "os": { + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81", + "codename": "Core" + } + }, + "agent": { + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", + "id": "a6147a6e-6626-4a84-9907-f372f6c61eee" + }, + "ecs": { + "version": "1.5.0" + }, + "service": { + "address": "kube-state-metrics:8080", + "type": "kubernetes" + } +} \ No newline at end of file diff --git a/test/packages/kubernetes/data_stream/state_storageclass/_dev/test/system/test-default-config.yml b/test/packages/kubernetes/data_stream/state_storageclass/_dev/test/system/test-default-config.yml new file mode 100644 index 0000000000..0f4bd620c0 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_storageclass/_dev/test/system/test-default-config.yml @@ -0,0 +1,6 @@ +service: kubernetes +data_stream: + vars: + hosts: + # this is the DNS name of the k8s service for kube-state-metrics deployment + - http://kube-state-metrics:8080 diff --git a/test/packages/kubernetes/data_stream/state_storageclass/agent/stream/stream.yml.hbs b/test/packages/kubernetes/data_stream/state_storageclass/agent/stream/stream.yml.hbs new file mode 100644 index 0000000000..acf1b2e2c6 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_storageclass/agent/stream/stream.yml.hbs @@ -0,0 +1,10 @@ +metricsets: ["state_storageclass"] +add_metadata: {{add_metadata}} +hosts: +{{#each hosts}} + - {{this}} +{{/each}} +period: {{period}} +{{#if leaderelection}} +condition: ${kubernetes_leaderelection.leader} == true +{{/if}} diff --git a/test/packages/kubernetes/data_stream/state_storageclass/fields/agent.yml b/test/packages/kubernetes/data_stream/state_storageclass/fields/agent.yml new file mode 100644 index 0000000000..da4e652c53 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_storageclass/fields/agent.yml @@ -0,0 +1,198 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + diff --git a/test/packages/kubernetes/data_stream/state_storageclass/fields/base-fields.yml b/test/packages/kubernetes/data_stream/state_storageclass/fields/base-fields.yml new file mode 100644 index 0000000000..d43ffd7964 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_storageclass/fields/base-fields.yml @@ -0,0 +1,90 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. +- name: kubernetes + type: group + fields: + - name: pod.name + type: keyword + description: > + Kubernetes pod name + + - name: pod.uid + type: keyword + description: > + Kubernetes pod UID + + - name: pod.ip + type: ip + description: > + Kubernetes pod IP + + - name: namespace + type: keyword + description: > + Kubernetes namespace + + - name: node.name + type: keyword + description: > + Kubernetes node name + + - name: node.hostname + type: keyword + description: > + Kubernetes hostname as reported by the node’s kernel + + - name: labels.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes labels map + + - name: annotations.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes annotations map + + - name: selectors.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes Service selectors map + + - name: replicaset.name + type: keyword + description: > + Kubernetes replicaset name + + - name: deployment.name + type: keyword + description: > + Kubernetes deployment name + + - name: statefulset.name + type: keyword + description: > + Kubernetes statefulset name + + - name: container.name + type: keyword + description: > + Kubernetes container name + + - name: container.image + type: keyword + description: >- + Kubernetes container image diff --git a/test/packages/kubernetes/data_stream/state_storageclass/fields/ecs.yml b/test/packages/kubernetes/data_stream/state_storageclass/fields/ecs.yml new file mode 100644 index 0000000000..cd4e3a89b1 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_storageclass/fields/ecs.yml @@ -0,0 +1,10 @@ +- external: ecs + name: ecs.version +- external: ecs + name: service.address +- external: ecs + name: service.type +- external: ecs + name: orchestrator.cluster.name +- external: ecs + name: orchestrator.cluster.url diff --git a/test/packages/kubernetes/data_stream/state_storageclass/fields/fields.yml b/test/packages/kubernetes/data_stream/state_storageclass/fields/fields.yml new file mode 100644 index 0000000000..6a0d31a627 --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_storageclass/fields/fields.yml @@ -0,0 +1,19 @@ +- name: kubernetes.storageclass + type: group + fields: + - name: name + dimension: true + type: keyword + description: Storage class name. + - name: provisioner + type: keyword + description: Volume provisioner for the storage class. + - name: reclaim_policy + type: keyword + description: Reclaim policy for dynamically created volumes + - name: volume_binding_mode + type: keyword + description: Mode for default provisioning and binding + - name: created + type: date + description: Storage class creation date diff --git a/test/packages/kubernetes/data_stream/state_storageclass/manifest.yml b/test/packages/kubernetes/data_stream/state_storageclass/manifest.yml new file mode 100644 index 0000000000..5e1821b11b --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_storageclass/manifest.yml @@ -0,0 +1,37 @@ +title: Kubernetes StorageClass metrics +type: metrics +streams: + - input: kubernetes/metrics + enabled: false + vars: + - name: add_metadata + type: bool + title: Add Metadata + multi: false + required: true + show_user: true + default: true + - name: hosts + type: text + title: Hosts + multi: true + required: true + show_user: true + default: + - kube-state-metrics:8080 + - name: leaderelection + type: bool + title: Leader Election + multi: false + required: true + show_user: true + default: true + - name: period + type: text + title: Period + multi: false + required: true + show_user: true + default: 10s + title: Kubernetes StorageClass metrics + description: Collect Kubernetes StorageClass metrics from kube_state_metrics diff --git a/test/packages/kubernetes/data_stream/state_storageclass/sample_event.json b/test/packages/kubernetes/data_stream/state_storageclass/sample_event.json new file mode 100644 index 0000000000..de074d381d --- /dev/null +++ b/test/packages/kubernetes/data_stream/state_storageclass/sample_event.json @@ -0,0 +1,59 @@ +{ + "@timestamp": "2020-06-25T12:39:44.399Z", + "agent": { + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "644323b5-5d6a-4dfb-92dd-35ca602db487", + "id": "a6147a6e-6626-4a84-9907-f372f6c61eee" + }, + "kubernetes": { + "storageclass": { + "provisioner": "k8s.io/minikube-hostpath", + "reclaim_policy": "Delete", + "volume_binding_mode": "Immediate", + "name": "standard", + "created": "2020-06-10T09:02:27.000Z" + }, + "labels": { + "addonmanager_kubernetes_io_mode": "EnsureExists" + } + }, + "host": { + "hostname": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "architecture": "x86_64", + "os": { + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81", + "codename": "Core" + }, + "id": "b0e83d397c054b8a99a431072fe4617b", + "name": "agent-ingest-management-clusterscope-674dbb75df-rp8cc", + "containerized": false, + "ip": [ + "172.17.0.11" + ], + "mac": [ + "02:42:ac:11:00:0b" + ] + }, + "event": { + "module": "kubernetes", + "duration": 5713503, + "dataset": "kubernetes.storageclass" + }, + "metricset": { + "name": "state_storageclass", + "period": 10000 + }, + "service": { + "address": "kube-state-metrics:8080", + "type": "kubernetes" + }, + "ecs": { + "version": "1.5.0" + } +} \ No newline at end of file diff --git a/test/packages/kubernetes/data_stream/system/_dev/deploy/k8s/.empty b/test/packages/kubernetes/data_stream/system/_dev/deploy/k8s/.empty new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/packages/kubernetes/data_stream/system/_dev/test/system/test-default-config.yml b/test/packages/kubernetes/data_stream/system/_dev/test/system/test-default-config.yml new file mode 100644 index 0000000000..6c69562d49 --- /dev/null +++ b/test/packages/kubernetes/data_stream/system/_dev/test/system/test-default-config.yml @@ -0,0 +1,7 @@ +service: kubernetes +data_stream: + vars: + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + period: 5s + hosts: + - https://{{Hostname}}:10250 diff --git a/test/packages/kubernetes/data_stream/system/agent/stream/stream.yml.hbs b/test/packages/kubernetes/data_stream/system/agent/stream/stream.yml.hbs new file mode 100644 index 0000000000..7157b762c7 --- /dev/null +++ b/test/packages/kubernetes/data_stream/system/agent/stream/stream.yml.hbs @@ -0,0 +1,11 @@ +metricsets: ["system"] +add_metadata: {{add_metadata}} +hosts: +{{#each hosts}} + - {{this}} +{{/each}} +period: {{period}} +{{#if bearer_token_file}} +bearer_token_file: {{bearer_token_file}} +ssl.verification_mode: {{ssl.verification_mode}} +{{/if}} diff --git a/test/packages/kubernetes/data_stream/system/fields/agent.yml b/test/packages/kubernetes/data_stream/system/fields/agent.yml new file mode 100644 index 0000000000..da4e652c53 --- /dev/null +++ b/test/packages/kubernetes/data_stream/system/fields/agent.yml @@ -0,0 +1,198 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + diff --git a/test/packages/kubernetes/data_stream/system/fields/base-fields.yml b/test/packages/kubernetes/data_stream/system/fields/base-fields.yml new file mode 100644 index 0000000000..0e3072d1ef --- /dev/null +++ b/test/packages/kubernetes/data_stream/system/fields/base-fields.yml @@ -0,0 +1,91 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. +- name: kubernetes + type: group + fields: + - name: pod.name + type: keyword + description: > + Kubernetes pod name + + - name: pod.uid + type: keyword + description: > + Kubernetes pod UID + + - name: pod.ip + type: ip + description: > + Kubernetes pod IP + + - name: namespace + type: keyword + description: > + Kubernetes namespace + + - name: node.name + dimension: true + type: keyword + description: > + Kubernetes node name + + - name: node.hostname + type: keyword + description: > + Kubernetes hostname as reported by the node’s kernel + + - name: labels.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes labels map + + - name: annotations.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes annotations map + + - name: selectors.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes Service selectors map + + - name: replicaset.name + type: keyword + description: > + Kubernetes replicaset name + + - name: deployment.name + type: keyword + description: > + Kubernetes deployment name + + - name: statefulset.name + type: keyword + description: > + Kubernetes statefulset name + + - name: container.name + type: keyword + description: > + Kubernetes container name + + - name: container.image + type: keyword + description: >- + Kubernetes container image diff --git a/test/packages/kubernetes/data_stream/system/fields/ecs.yml b/test/packages/kubernetes/data_stream/system/fields/ecs.yml new file mode 100644 index 0000000000..cd4e3a89b1 --- /dev/null +++ b/test/packages/kubernetes/data_stream/system/fields/ecs.yml @@ -0,0 +1,10 @@ +- external: ecs + name: ecs.version +- external: ecs + name: service.address +- external: ecs + name: service.type +- external: ecs + name: orchestrator.cluster.name +- external: ecs + name: orchestrator.cluster.url diff --git a/test/packages/kubernetes/data_stream/system/fields/fields.yml b/test/packages/kubernetes/data_stream/system/fields/fields.yml new file mode 100644 index 0000000000..65fc48d0dd --- /dev/null +++ b/test/packages/kubernetes/data_stream/system/fields/fields.yml @@ -0,0 +1,74 @@ +- name: kubernetes.system + type: group + fields: + - name: container + dimension: true + type: keyword + description: | + Container name + - name: start_time + type: date + description: | + Start time + - name: cpu + type: group + fields: + - name: usage + type: group + fields: + - name: core + type: group + fields: + - name: ns + type: long + metric_type: gauge + description: | + CPU Core usage nanoseconds + - name: nanocores + type: long + metric_type: gauge + description: | + CPU used nanocores + - name: memory + type: group + fields: + - name: usage + type: group + fields: + - name: bytes + type: long + format: bytes + unit: byte + metric_type: gauge + description: | + Total memory usage + - name: rss + type: group + fields: + - name: bytes + type: long + format: bytes + unit: byte + metric_type: gauge + description: | + RSS memory usage + - name: workingset + type: group + fields: + - name: bytes + type: long + format: bytes + unit: byte + metric_type: gauge + description: | + Working set memory usage + - name: pagefaults + type: long + metric_type: counter + description: | + Number of page faults + - name: majorpagefaults + type: long + metric_type: counter + description: | + Number of major page faults diff --git a/test/packages/kubernetes/data_stream/system/manifest.yml b/test/packages/kubernetes/data_stream/system/manifest.yml new file mode 100644 index 0000000000..322c0958af --- /dev/null +++ b/test/packages/kubernetes/data_stream/system/manifest.yml @@ -0,0 +1,43 @@ +title: Kubernetes System metrics +type: metrics +streams: + - input: kubernetes/metrics + vars: + - name: add_metadata + type: bool + title: Add Metadata + multi: false + required: true + show_user: true + default: true + - name: bearer_token_file + type: text + title: Bearer Token File + multi: false + required: true + show_user: true + default: /var/run/secrets/kubernetes.io/serviceaccount/token + - name: hosts + type: text + title: Hosts + multi: true + required: true + show_user: true + default: + - https://${env.NODE_NAME}:10250 + - name: period + type: text + title: Period + multi: false + required: true + show_user: true + default: 10s + - name: ssl.verification_mode + type: text + title: SSL Verification Mode + multi: false + required: true + show_user: true + default: none + title: Kubernetes System metrics + description: Collect Kubernetes system metrics diff --git a/test/packages/kubernetes/data_stream/system/sample_event.json b/test/packages/kubernetes/data_stream/system/sample_event.json new file mode 100644 index 0000000000..1c490ef754 --- /dev/null +++ b/test/packages/kubernetes/data_stream/system/sample_event.json @@ -0,0 +1,101 @@ +{ + "@timestamp": "2020-06-25T12:39:59.647Z", + "service": { + "address": "minikube:10250", + "type": "kubernetes" + }, + "event": { + "duration": 20012905, + "dataset": "kubernetes.system", + "module": "kubernetes" + }, + "ecs": { + "version": "1.5.0" + }, + "host": { + "mac": [ + "aa:83:2f:7f:6b:12", + "02:42:d4:8c:94:93", + "2a:59:80:9e:fc:d6", + "da:3a:d9:e9:70:52", + "8a:0a:b6:18:ba:76", + "f6:47:fa:80:e8:8b", + "9e:c3:ff:95:e4:8e", + "6e:1c:29:50:d4:0c", + "b6:f3:11:60:14:ed", + "22:f2:2a:96:1e:7b", + "56:34:ba:de:57:20", + "aa:78:91:29:81:f7" + ], + "hostname": "minikube", + "name": "minikube", + "architecture": "x86_64", + "os": { + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81", + "codename": "Core", + "platform": "centos" + }, + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false, + "ip": [ + "192.168.64.10", + "fe80::a883:2fff:fe7f:6b12", + "172.17.0.1", + "fe80::42:d4ff:fe8c:9493", + "fe80::2859:80ff:fe9e:fcd6", + "fe80::d83a:d9ff:fee9:7052", + "fe80::880a:b6ff:fe18:ba76", + "fe80::f447:faff:fe80:e88b", + "fe80::9cc3:ffff:fe95:e48e", + "fe80::6c1c:29ff:fe50:d40c", + "fe80::b4f3:11ff:fe60:14ed", + "fe80::20f2:2aff:fe96:1e7b", + "fe80::5434:baff:fede:5720", + "fe80::a878:91ff:fe29:81f7" + ] + }, + "agent": { + "ephemeral_id": "b964a246-96c0-456a-a5c2-8c8b1040ecaf", + "id": "f7ec69f9-4997-4e76-b6c7-0c75206b727a", + "name": "minikube", + "type": "metricbeat", + "version": "8.0.0" + }, + "kubernetes": { + "node": { + "name": "minikube" + }, + "system": { + "container": "runtime", + "cpu": { + "usage": { + "nanocores": 35779815, + "core": { + "ns": 530899961233 + } + } + }, + "memory": { + "pagefaults": 12944019, + "majorpagefaults": 99, + "usage": { + "bytes": 198279168 + }, + "workingset": { + "bytes": 178794496 + }, + "rss": { + "bytes": 125259776 + } + }, + "start_time": "2020-06-25T07:19:32Z" + } + }, + "metricset": { + "name": "system", + "period": 10000 + } +} \ No newline at end of file diff --git a/test/packages/kubernetes/data_stream/volume/_dev/deploy/k8s/.empty b/test/packages/kubernetes/data_stream/volume/_dev/deploy/k8s/.empty new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/packages/kubernetes/data_stream/volume/_dev/test/system/test-default-config.yml b/test/packages/kubernetes/data_stream/volume/_dev/test/system/test-default-config.yml new file mode 100644 index 0000000000..6c69562d49 --- /dev/null +++ b/test/packages/kubernetes/data_stream/volume/_dev/test/system/test-default-config.yml @@ -0,0 +1,7 @@ +service: kubernetes +data_stream: + vars: + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + period: 5s + hosts: + - https://{{Hostname}}:10250 diff --git a/test/packages/kubernetes/data_stream/volume/agent/stream/stream.yml.hbs b/test/packages/kubernetes/data_stream/volume/agent/stream/stream.yml.hbs new file mode 100644 index 0000000000..d546a66630 --- /dev/null +++ b/test/packages/kubernetes/data_stream/volume/agent/stream/stream.yml.hbs @@ -0,0 +1,11 @@ +metricsets: ["volume"] +add_metadata: {{add_metadata}} +hosts: +{{#each hosts}} + - {{this}} +{{/each}} +period: {{period}} +{{#if bearer_token_file}} +bearer_token_file: {{bearer_token_file}} +ssl.verification_mode: {{ssl.verification_mode}} +{{/if}} diff --git a/test/packages/kubernetes/data_stream/volume/fields/agent.yml b/test/packages/kubernetes/data_stream/volume/fields/agent.yml new file mode 100644 index 0000000000..da4e652c53 --- /dev/null +++ b/test/packages/kubernetes/data_stream/volume/fields/agent.yml @@ -0,0 +1,198 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + diff --git a/test/packages/kubernetes/data_stream/volume/fields/base-fields.yml b/test/packages/kubernetes/data_stream/volume/fields/base-fields.yml new file mode 100644 index 0000000000..d43ffd7964 --- /dev/null +++ b/test/packages/kubernetes/data_stream/volume/fields/base-fields.yml @@ -0,0 +1,90 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. +- name: kubernetes + type: group + fields: + - name: pod.name + type: keyword + description: > + Kubernetes pod name + + - name: pod.uid + type: keyword + description: > + Kubernetes pod UID + + - name: pod.ip + type: ip + description: > + Kubernetes pod IP + + - name: namespace + type: keyword + description: > + Kubernetes namespace + + - name: node.name + type: keyword + description: > + Kubernetes node name + + - name: node.hostname + type: keyword + description: > + Kubernetes hostname as reported by the node’s kernel + + - name: labels.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes labels map + + - name: annotations.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes annotations map + + - name: selectors.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Kubernetes Service selectors map + + - name: replicaset.name + type: keyword + description: > + Kubernetes replicaset name + + - name: deployment.name + type: keyword + description: > + Kubernetes deployment name + + - name: statefulset.name + type: keyword + description: > + Kubernetes statefulset name + + - name: container.name + type: keyword + description: > + Kubernetes container name + + - name: container.image + type: keyword + description: >- + Kubernetes container image diff --git a/test/packages/kubernetes/data_stream/volume/fields/ecs.yml b/test/packages/kubernetes/data_stream/volume/fields/ecs.yml new file mode 100644 index 0000000000..cd4e3a89b1 --- /dev/null +++ b/test/packages/kubernetes/data_stream/volume/fields/ecs.yml @@ -0,0 +1,10 @@ +- external: ecs + name: ecs.version +- external: ecs + name: service.address +- external: ecs + name: service.type +- external: ecs + name: orchestrator.cluster.name +- external: ecs + name: orchestrator.cluster.url diff --git a/test/packages/kubernetes/data_stream/volume/fields/fields.yml b/test/packages/kubernetes/data_stream/volume/fields/fields.yml new file mode 100644 index 0000000000..afebbf228d --- /dev/null +++ b/test/packages/kubernetes/data_stream/volume/fields/fields.yml @@ -0,0 +1,65 @@ +- name: kubernetes.volume + type: group + fields: + - name: name + dimension: true + type: keyword + description: | + Volume name + - name: fs + type: group + fields: + - name: capacity + type: group + fields: + - name: bytes + type: long + format: bytes + unit: byte + metric_type: gauge + description: | + Filesystem total capacity in bytes + - name: available + type: group + fields: + - name: bytes + type: long + format: bytes + unit: byte + metric_type: gauge + description: | + Filesystem total available in bytes + - name: used + type: group + fields: + - name: bytes + type: long + format: bytes + unit: byte + metric_type: gauge + description: | + Filesystem total used in bytes + - name: pct + type: long + unit: percent + metric_type: gauge + description: | + Percentage of filesystem total used + - name: inodes + type: group + fields: + - name: used + type: long + metric_type: gauge + description: | + Used inodes + - name: free + type: long + metric_type: gauge + description: | + Free inodes + - name: count + type: long + metric_type: gauge + description: | + Total inodes diff --git a/test/packages/kubernetes/data_stream/volume/manifest.yml b/test/packages/kubernetes/data_stream/volume/manifest.yml new file mode 100644 index 0000000000..6f36b86dcb --- /dev/null +++ b/test/packages/kubernetes/data_stream/volume/manifest.yml @@ -0,0 +1,43 @@ +title: Kubernetes Volume metrics +type: metrics +streams: + - input: kubernetes/metrics + vars: + - name: add_metadata + type: bool + title: Add Metadata + multi: false + required: true + show_user: true + default: true + - name: bearer_token_file + type: text + title: Bearer Token File + multi: false + required: true + show_user: true + default: /var/run/secrets/kubernetes.io/serviceaccount/token + - name: hosts + type: text + title: Hosts + multi: true + required: true + show_user: true + default: + - https://${env.NODE_NAME}:10250 + - name: period + type: text + title: Period + multi: false + required: true + show_user: true + default: 10s + - name: ssl.verification_mode + type: text + title: SSL Verification Mode + multi: false + required: true + show_user: true + default: none + title: Kubernetes Volume metrics + description: Collect Kubernetes Volume metrics diff --git a/test/packages/kubernetes/data_stream/volume/sample_event.json b/test/packages/kubernetes/data_stream/volume/sample_event.json new file mode 100644 index 0000000000..700d089f1b --- /dev/null +++ b/test/packages/kubernetes/data_stream/volume/sample_event.json @@ -0,0 +1,99 @@ +{ + "@timestamp": "2020-06-25T12:40:19.649Z", + "ecs": { + "version": "1.5.0" + }, + "metricset": { + "name": "volume", + "period": 10000 + }, + "service": { + "type": "kubernetes", + "address": "minikube:10250" + }, + "kubernetes": { + "pod": { + "name": "metricbeat-g9fc6" + }, + "volume": { + "name": "config", + "fs": { + "inodes": { + "used": 5, + "free": 9549949, + "count": 9768928 + }, + "available": { + "bytes": 7719858176 + }, + "capacity": { + "bytes": 17361141760 + }, + "used": { + "bytes": 12288 + } + } + }, + "namespace": "kube-system", + "node": { + "name": "minikube" + } + }, + "host": { + "architecture": "x86_64", + "os": { + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.19.81", + "codename": "Core" + }, + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false, + "ip": [ + "192.168.64.10", + "fe80::a883:2fff:fe7f:6b12", + "172.17.0.1", + "fe80::42:d4ff:fe8c:9493", + "fe80::2859:80ff:fe9e:fcd6", + "fe80::d83a:d9ff:fee9:7052", + "fe80::880a:b6ff:fe18:ba76", + "fe80::f447:faff:fe80:e88b", + "fe80::9cc3:ffff:fe95:e48e", + "fe80::6c1c:29ff:fe50:d40c", + "fe80::b4f3:11ff:fe60:14ed", + "fe80::20f2:2aff:fe96:1e7b", + "fe80::5434:baff:fede:5720", + "fe80::a878:91ff:fe29:81f7" + ], + "name": "minikube", + "mac": [ + "aa:83:2f:7f:6b:12", + "02:42:d4:8c:94:93", + "2a:59:80:9e:fc:d6", + "da:3a:d9:e9:70:52", + "8a:0a:b6:18:ba:76", + "f6:47:fa:80:e8:8b", + "9e:c3:ff:95:e4:8e", + "6e:1c:29:50:d4:0c", + "b6:f3:11:60:14:ed", + "22:f2:2a:96:1e:7b", + "56:34:ba:de:57:20", + "aa:78:91:29:81:f7" + ], + "hostname": "minikube" + }, + "agent": { + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "b964a246-96c0-456a-a5c2-8c8b1040ecaf", + "id": "f7ec69f9-4997-4e76-b6c7-0c75206b727a", + "name": "minikube" + }, + "event": { + "dataset": "kubernetes.volume", + "module": "kubernetes", + "duration": 12481688 + } +} \ No newline at end of file diff --git a/test/packages/kubernetes/docs/README.md b/test/packages/kubernetes/docs/README.md index 5a2d112c79..8713649963 100644 --- a/test/packages/kubernetes/docs/README.md +++ b/test/packages/kubernetes/docs/README.md @@ -1,4 +1,93 @@ # Kubernetes integration -This integration is used to collect metrics from -[Kubernetes clusters](https://kubernetes.io/). \ No newline at end of file +This integration is used to collect logs and metrics from +[Kubernetes clusters](https://kubernetes.io/). + +As one of the main pieces provided for Kubernetes monitoring, this integration is capable of fetching metrics from several components: + +- [kubelet](https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/) +- [kube-state-metrics](https://github.com/kubernetes/kube-state-metrics) +- [apiserver](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/) +- [controller-manager](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-controller-manager/) +- [scheduler](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-scheduler/) +- [proxy](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-proxy/) + +Some of the previous components are running on each of the Kubernetes nodes (like `kubelet` or `proxy`) while others provide +a single cluster-wide endpoint. This is important to determine the optimal configuration and running strategy +for the different datasets included in the integration. + + +#### Kubernetes endpoints and metricsets + +Kubernetes module is a bit complex as its internal datasets require access to a wide variety of endpoints. + +This section highlights and introduces some groups of datasets with similar endpoint access needs. +For more details on the datasets see `configuration example` and the `datasets` sections below. + + +#### node / system / pod / container / module / volume + +The datasets `container`, `node`, `pod`, `system` and `volume` require access to the `kubelet endpoint` in each of +the Kubernetes nodes, hence it's recommended to include them as part +of an `Agent DaemonSet` or standalone Agents running on the hosts. + +Depending on the version and configuration of Kubernetes nodes, `kubelet` might provide a read only http port (typically 10255), +which is used in some configuration examples. But in general, and lately, this endpoint requires SSL (`https`) access +(to port 10250 by default) and token based authentication. + + +##### state_* and event + +All datasets with the `state_` prefix require `hosts` field pointing to `kube-state-metrics` +service within the cluster. As the service provides cluster-wide metrics, there's no need to fetch them per node, +hence the recommendation is to run these datasets as part of an `Agent Deployment` with one only replica. + +Note: Kube-state-metrics is not deployed by default in Kubernetes. For these cases the instructions for its +deployment are available [here](https://github.com/kubernetes/kube-state-metrics#kubernetes-deployment). +Generally `kube-state-metrics` runs a `Deployment` and is accessible via a service called `kube-state-metrics` on +`kube-system` namespace, which will be the service to use in our configuration. + +state_* datasets are not enabled by default. + +#### apiserver + +The apiserver dataset requires access to the Kubernetes API, which should be easily available in all Kubernetes +environments. Depending on the Kubernetes configuration, the API access might require SSL (`https`) and token +based authentication. + +#### proxy + +The proxy dataset requires access to the proxy endpoint in each of Kubernetes nodes, hence it's recommended +to configure it as a part of an `Agent DaemonSet`. + +#### scheduler and controllermanager + +These datasets require access to the Kubernetes `controller-manager` and `scheduler` endpoints. By default, these pods +run only on master nodes, and they are not exposed via a Service, but there are different strategies +available for its configuration: + +- Create `Kubernetes Services` to make `kube-controller-manager` and `kube-scheduler` available and configure + the datasets to point to these services as part of an `Agent Deployment`. +- Run these datasets as part an `Agent Daemonset` (with HostNetwork setting) with a `nodeSelector` to only run on Master nodes. + +These datasets are not enabled by default. + +Note: In some "As a Service" Kubernetes implementations, like `GKE`, the master nodes or even the pods running on +the masters won't be visible. In these cases it won't be possible to use `scheduler` and `controllermanager` metricsets. + +#### container-logs + +The container-logs dataset requires access to the log files in each Kubernetes node where the container logs are stored. +This defaults to `/var/log/containers/*${kubernetes.container.id}.log`. + +## Compatibility + +The Kubernetes package is tested with Kubernetes 1.13.x, 1.14.x, 1.15.x, 1.16.x, 1.17.x, and 1.18.x + +## Dashboard + +Kubernetes integration is shipped including default dashboards for `apiserver`, `controllermanager`, `overview`, `proxy` and `scheduler`. + +If you are using HA for those components, be aware that when gathering data from all instances the dashboard will usually show the average of the metrics. For those scenarios filtering by hosts or service address is possible. + +Cluster selector in `overview` dashboard helps in distinguishing and filtering metrics collected from multiple clusters. If you want to focus on a subset of the Kubernetes clusters for monitoring a specific scenario, this cluster selector could be a handy tool. Note that this selector gets populated from the `orchestrator.cluster.name` field that may not always be available. This field gets its value from sources like `kube_config`, `kubeadm-config` configMap, and Google Cloud's meta API for GKE. If the sources mentioned above don't provide this value, metricbeat will not report it. However, you can always use [processors](https://www.elastic.co/guide/en/beats/metricbeat/current/defining-processors.html) to set this field and utilize it in the `cluster overview` dashboard. diff --git a/test/packages/kubernetes/docs/container-logs.md b/test/packages/kubernetes/docs/container-logs.md new file mode 100644 index 0000000000..3595e1c268 --- /dev/null +++ b/test/packages/kubernetes/docs/container-logs.md @@ -0,0 +1,6 @@ +# container-logs + +container-logs integration collects and parses logs of Kubernetes containers. + +It requires access to the log files in each Kubernetes node where the container logs are stored. +This defaults to `/var/log/containers/*${kubernetes.container.id}.log`. \ No newline at end of file diff --git a/test/packages/kubernetes/docs/events.md b/test/packages/kubernetes/docs/events.md index 1ea6c84f6c..740537baac 100644 --- a/test/packages/kubernetes/docs/events.md +++ b/test/packages/kubernetes/docs/events.md @@ -115,7 +115,7 @@ An example event for `event` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | data_stream.type | Data stream type. | constant_keyword | | -| ecs.version | ECS version | keyword | | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | | host.architecture | Operating system architecture. | keyword | | | host.containerized | If the host is a container. | boolean | | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | @@ -141,7 +141,7 @@ An example event for `event` looks as following: | kubernetes.event.involved_object.kind | API kind of the object | keyword | | | kubernetes.event.involved_object.name | name of the object | keyword | | | kubernetes.event.involved_object.resource_version | resource version of the object | keyword | | -| kubernetes.event.involved_object.uid | UUID version of the object | keyword | | +| kubernetes.event.involved_object.uid | uid version of the object | keyword | | | kubernetes.event.message | Message recorded for the given event | text | | | kubernetes.event.metadata.generate_name | Generate name of the event | keyword | | | kubernetes.event.metadata.name | Name of the event | keyword | | @@ -168,4 +168,4 @@ An example event for `event` looks as following: | kubernetes.statefulset.name | Kubernetes statefulset name | keyword | | | orchestrator.cluster.name | Name of the cluster. | keyword | | | orchestrator.cluster.url | URL of the API used to manage the cluster. | keyword | | -| service.type | Service type | keyword | | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | diff --git a/test/packages/kubernetes/docs/kube-apiserver.md b/test/packages/kubernetes/docs/kube-apiserver.md index 52a5bec309..df9cf222ea 100644 --- a/test/packages/kubernetes/docs/kube-apiserver.md +++ b/test/packages/kubernetes/docs/kube-apiserver.md @@ -120,7 +120,7 @@ An example event for `apiserver` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | | data_stream.type | Data stream type. | constant_keyword | | | -| ecs.version | ECS version | keyword | | | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | | | host.architecture | Operating system architecture. | keyword | | | | host.containerized | If the host is a container. | boolean | | | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | | @@ -182,5 +182,5 @@ An example event for `apiserver` looks as following: | kubernetes.apiserver.request.version | version for the group | keyword | | | | orchestrator.cluster.name | Name of the cluster. | keyword | | | | orchestrator.cluster.url | URL of the API used to manage the cluster. | keyword | | | -| service.address | Service address | keyword | | | -| service.type | Service type | keyword | | | +| service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | | diff --git a/test/packages/kubernetes/docs/kube-controller-manager.md b/test/packages/kubernetes/docs/kube-controller-manager.md index f987fdc6bb..4215f143bb 100644 --- a/test/packages/kubernetes/docs/kube-controller-manager.md +++ b/test/packages/kubernetes/docs/kube-controller-manager.md @@ -126,7 +126,7 @@ An example event for `controllermanager` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | | data_stream.type | Data stream type. | constant_keyword | | | -| ecs.version | ECS version | keyword | | | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | | | host.architecture | Operating system architecture. | keyword | | | | host.containerized | If the host is a container. | boolean | | | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | | @@ -191,5 +191,5 @@ An example event for `controllermanager` looks as following: | kubernetes.statefulset.name | Kubernetes statefulset name | keyword | | | | orchestrator.cluster.name | Name of the cluster. | keyword | | | | orchestrator.cluster.url | URL of the API used to manage the cluster. | keyword | | | -| service.address | Service address | keyword | | | -| service.type | Service type | keyword | | | +| service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | | diff --git a/test/packages/kubernetes/docs/kube-proxy.md b/test/packages/kubernetes/docs/kube-proxy.md index 675550ee6c..5d4aa32e32 100644 --- a/test/packages/kubernetes/docs/kube-proxy.md +++ b/test/packages/kubernetes/docs/kube-proxy.md @@ -249,7 +249,7 @@ An example event for `proxy` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | | data_stream.type | Data stream type. | constant_keyword | | | -| ecs.version | ECS version | keyword | | | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | | | host.architecture | Operating system architecture. | keyword | | | | host.containerized | If the host is a container. | boolean | | | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | | @@ -294,5 +294,5 @@ An example event for `proxy` looks as following: | kubernetes.proxy.sync.rules.duration.us.sum | SyncProxyRules duration, sum of durations in microseconds | long | | counter | | orchestrator.cluster.name | Name of the cluster. | keyword | | | | orchestrator.cluster.url | URL of the API used to manage the cluster. | keyword | | | -| service.address | Service address | keyword | | | -| service.type | Service type | keyword | | | +| service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | | diff --git a/test/packages/kubernetes/docs/kube-scheduler.md b/test/packages/kubernetes/docs/kube-scheduler.md index 5c239728f9..7d8f8d5ead 100644 --- a/test/packages/kubernetes/docs/kube-scheduler.md +++ b/test/packages/kubernetes/docs/kube-scheduler.md @@ -112,7 +112,7 @@ An example event for `scheduler` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | | data_stream.type | Data stream type. | constant_keyword | | | -| ecs.version | ECS version | keyword | | | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | | | host.architecture | Operating system architecture. | keyword | | | | host.containerized | If the host is a container. | boolean | | | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | | @@ -179,5 +179,5 @@ An example event for `scheduler` looks as following: | kubernetes.statefulset.name | Kubernetes statefulset name | keyword | | | | orchestrator.cluster.name | Name of the cluster. | keyword | | | | orchestrator.cluster.url | URL of the API used to manage the cluster. | keyword | | | -| service.address | Service address | keyword | | | -| service.type | Service type | keyword | | | +| service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | | diff --git a/test/packages/kubernetes/docs/kube-state-metrics.md b/test/packages/kubernetes/docs/kube-state-metrics.md index e94360c85e..1c998e2c9f 100644 --- a/test/packages/kubernetes/docs/kube-state-metrics.md +++ b/test/packages/kubernetes/docs/kube-state-metrics.md @@ -114,11 +114,11 @@ An example event for `state_container` looks as following: | container.image.name | Name of the image the container was built on. | keyword | | | | container.labels | Image labels. | object | | | | container.name | Container name. | keyword | | | -| container.runtime | Runtime managing this container | keyword | | | +| container.runtime | Runtime managing this container. | keyword | | | | data_stream.dataset | Data stream dataset. | constant_keyword | | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | | data_stream.type | Data stream type. | constant_keyword | | | -| ecs.version | ECS version | keyword | | | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | | | host.architecture | Operating system architecture. | keyword | | | | host.containerized | If the host is a container. | boolean | | | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | | @@ -163,8 +163,8 @@ An example event for `state_container` looks as following: | kubernetes.statefulset.name | Kubernetes statefulset name | keyword | | | | orchestrator.cluster.name | Name of the cluster. | keyword | | | | orchestrator.cluster.url | URL of the API used to manage the cluster. | keyword | | | -| service.address | Service address | keyword | | | -| service.type | Service type | keyword | | | +| service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | | ### state_cronjob @@ -264,7 +264,7 @@ An example event for `state_cronjob` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | | data_stream.type | Data stream type. | constant_keyword | | | -| ecs.version | ECS version | keyword | | | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | | | host.architecture | Operating system architecture. | keyword | | | | host.containerized | If the host is a container. | boolean | | | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | | @@ -306,8 +306,8 @@ An example event for `state_cronjob` looks as following: | kubernetes.statefulset.name | Kubernetes statefulset name | keyword | | | | orchestrator.cluster.name | Name of the cluster. | keyword | | | | orchestrator.cluster.url | URL of the API used to manage the cluster. | keyword | | | -| service.address | Service address | keyword | | | -| service.type | Service type | keyword | | | +| service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | | ### state_daemonset @@ -403,7 +403,7 @@ An example event for `state_daemonset` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | data_stream.type | Data stream type. | constant_keyword | | -| ecs.version | ECS version | keyword | | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | | host.architecture | Operating system architecture. | keyword | | | host.containerized | If the host is a container. | boolean | | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | @@ -441,8 +441,8 @@ An example event for `state_daemonset` looks as following: | kubernetes.statefulset.name | Kubernetes statefulset name | keyword | | | orchestrator.cluster.name | Name of the cluster. | keyword | | | orchestrator.cluster.url | URL of the API used to manage the cluster. | keyword | | -| service.address | Service address | keyword | | -| service.type | Service type | keyword | | +| service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | ### state_deployment @@ -539,7 +539,7 @@ An example event for `state_deployment` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | data_stream.type | Data stream type. | constant_keyword | | -| ecs.version | ECS version | keyword | | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | | host.architecture | Operating system architecture. | keyword | | | host.containerized | If the host is a container. | boolean | | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | @@ -577,8 +577,8 @@ An example event for `state_deployment` looks as following: | kubernetes.statefulset.name | Kubernetes statefulset name | keyword | | | orchestrator.cluster.name | Name of the cluster. | keyword | | | orchestrator.cluster.url | URL of the API used to manage the cluster. | keyword | | -| service.address | Service address | keyword | | -| service.type | Service type | keyword | | +| service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | ### state_job @@ -684,7 +684,7 @@ An example event for `state_job` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | data_stream.type | Data stream type. | constant_keyword | | -| ecs.version | ECS version | keyword | | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | | host.architecture | Operating system architecture. | keyword | | | host.containerized | If the host is a container. | boolean | | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | @@ -730,8 +730,8 @@ An example event for `state_job` looks as following: | kubernetes.statefulset.name | Kubernetes statefulset name | keyword | | | orchestrator.cluster.name | Name of the cluster. | keyword | | | orchestrator.cluster.url | URL of the API used to manage the cluster. | keyword | | -| service.address | Service address | keyword | | -| service.type | Service type | keyword | | +| service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | ### state_node @@ -853,7 +853,7 @@ An example event for `state_node` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | | data_stream.type | Data stream type. | constant_keyword | | | -| ecs.version | ECS version | keyword | | | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | | | host.architecture | Operating system architecture. | keyword | | | | host.containerized | If the host is a container. | boolean | | | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | | @@ -898,8 +898,8 @@ An example event for `state_node` looks as following: | kubernetes.statefulset.name | Kubernetes statefulset name | keyword | | | | orchestrator.cluster.name | Name of the cluster. | keyword | | | | orchestrator.cluster.url | URL of the API used to manage the cluster. | keyword | | | -| service.address | Service address | keyword | | | -| service.type | Service type | keyword | | | +| service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | | ### state_persistentvolume @@ -993,7 +993,7 @@ An example event for `state_persistentvolume` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | | data_stream.type | Data stream type. | constant_keyword | | | -| ecs.version | ECS version | keyword | | | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | | | host.architecture | Operating system architecture. | keyword | | | | host.containerized | If the host is a container. | boolean | | | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | | @@ -1030,8 +1030,8 @@ An example event for `state_persistentvolume` looks as following: | kubernetes.statefulset.name | Kubernetes statefulset name | keyword | | | | orchestrator.cluster.name | Name of the cluster. | keyword | | | | orchestrator.cluster.url | URL of the API used to manage the cluster. | keyword | | | -| service.address | Service address | keyword | | | -| service.type | Service type | keyword | | | +| service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | | ### state_persistentvolumeclaim @@ -1125,7 +1125,7 @@ An example event for `state_persistentvolumeclaim` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | | data_stream.type | Data stream type. | constant_keyword | | | -| ecs.version | ECS version | keyword | | | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | | | host.architecture | Operating system architecture. | keyword | | | | host.containerized | If the host is a container. | boolean | | | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | | @@ -1164,8 +1164,8 @@ An example event for `state_persistentvolumeclaim` looks as following: | kubernetes.statefulset.name | Kubernetes statefulset name | keyword | | | | orchestrator.cluster.name | Name of the cluster. | keyword | | | | orchestrator.cluster.url | URL of the API used to manage the cluster. | keyword | | | -| service.address | Service address | keyword | | | -| service.type | Service type | keyword | | | +| service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | | ### state_pod @@ -1265,11 +1265,11 @@ An example event for `state_pod` looks as following: | container.image.name | Name of the image the container was built on. | keyword | | container.labels | Image labels. | object | | container.name | Container name. | keyword | -| container.runtime | Runtime managing this container | keyword | +| container.runtime | Runtime managing this container. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version | keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -1307,8 +1307,8 @@ An example event for `state_pod` looks as following: | kubernetes.statefulset.name | Kubernetes statefulset name | keyword | | orchestrator.cluster.name | Name of the cluster. | keyword | | orchestrator.cluster.url | URL of the API used to manage the cluster. | keyword | -| service.address | Service address | keyword | -| service.type | Service type | keyword | +| service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | ### state_replicaset @@ -1411,7 +1411,7 @@ An example event for `state_replicaset` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | data_stream.type | Data stream type. | constant_keyword | | -| ecs.version | ECS version | keyword | | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | | host.architecture | Operating system architecture. | keyword | | | host.containerized | If the host is a container. | boolean | | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | @@ -1449,8 +1449,8 @@ An example event for `state_replicaset` looks as following: | kubernetes.statefulset.name | Kubernetes statefulset name | keyword | | | orchestrator.cluster.name | Name of the cluster. | keyword | | | orchestrator.cluster.url | URL of the API used to manage the cluster. | keyword | | -| service.address | Service address | keyword | | -| service.type | Service type | keyword | | +| service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | ### state_resourcequota @@ -1540,7 +1540,7 @@ An example event for `state_resourcequota` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | | data_stream.type | Data stream type. | constant_keyword | | | -| ecs.version | ECS version | keyword | | | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | | | host.architecture | Operating system architecture. | keyword | | | | host.containerized | If the host is a container. | boolean | | | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | | @@ -1578,8 +1578,8 @@ An example event for `state_resourcequota` looks as following: | kubernetes.statefulset.name | Kubernetes statefulset name | keyword | | | | orchestrator.cluster.name | Name of the cluster. | keyword | | | | orchestrator.cluster.url | URL of the API used to manage the cluster. | keyword | | | -| service.address | Service address | keyword | | | -| service.type | Service type | keyword | | | +| service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | | ### state_service @@ -1675,7 +1675,7 @@ An example event for `state_service` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version | keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -1705,7 +1705,7 @@ An example event for `state_service` looks as following: | kubernetes.pod.uid | Kubernetes pod UID | keyword | | kubernetes.replicaset.name | Kubernetes replicaset name | keyword | | kubernetes.selectors.\* | Kubernetes Service selectors map | object | -| kubernetes.service.cluster_ip | Internal IP for the service. | ip | +| kubernetes.service.cluster_ip | Internal IP for the service. | keyword | | kubernetes.service.created | Service creation date | date | | kubernetes.service.external_ip | Service external IP | keyword | | kubernetes.service.external_name | Service external DNS name | keyword | @@ -1717,8 +1717,8 @@ An example event for `state_service` looks as following: | kubernetes.statefulset.name | Kubernetes statefulset name | keyword | | orchestrator.cluster.name | Name of the cluster. | keyword | | orchestrator.cluster.url | URL of the API used to manage the cluster. | keyword | -| service.address | Service address | keyword | -| service.type | Service type | keyword | +| service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | ### state_statefulset @@ -1813,7 +1813,7 @@ An example event for `state_statefulset` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | data_stream.type | Data stream type. | constant_keyword | | -| ecs.version | ECS version | keyword | | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | | host.architecture | Operating system architecture. | keyword | | | host.containerized | If the host is a container. | boolean | | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | @@ -1852,8 +1852,8 @@ An example event for `state_statefulset` looks as following: | kubernetes.statefulset.replicas.ready | The number of ready replicas per StatefulSet | long | gauge | | orchestrator.cluster.name | Name of the cluster. | keyword | | | orchestrator.cluster.url | URL of the API used to manage the cluster. | keyword | | -| service.address | Service address | keyword | | -| service.type | Service type | keyword | | +| service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | ### state_storageclass @@ -1946,7 +1946,7 @@ An example event for `state_storageclass` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version | keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -1984,5 +1984,5 @@ An example event for `state_storageclass` looks as following: | kubernetes.storageclass.volume_binding_mode | Mode for default provisioning and binding | keyword | | orchestrator.cluster.name | Name of the cluster. | keyword | | orchestrator.cluster.url | URL of the API used to manage the cluster. | keyword | -| service.address | Service address | keyword | -| service.type | Service type | keyword | +| service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | diff --git a/test/packages/kubernetes/docs/kubelet.md b/test/packages/kubernetes/docs/kubelet.md index d92fbe4ad5..16de644c22 100644 --- a/test/packages/kubernetes/docs/kubelet.md +++ b/test/packages/kubernetes/docs/kubelet.md @@ -183,7 +183,7 @@ An example event for `container` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | | data_stream.type | Data stream type. | constant_keyword | | | -| ecs.version | ECS version | keyword | | | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | | | host.architecture | Operating system architecture. | keyword | | | | host.containerized | If the host is a container. | boolean | | | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | | @@ -240,8 +240,8 @@ An example event for `container` looks as following: | kubernetes.statefulset.name | Kubernetes statefulset name | keyword | | | | orchestrator.cluster.name | Name of the cluster. | keyword | | | | orchestrator.cluster.url | URL of the API used to manage the cluster. | keyword | | | -| service.address | Service address | keyword | | | -| service.type | Service type | keyword | | | +| service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | | ### node @@ -423,7 +423,7 @@ An example event for `node` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | | data_stream.type | Data stream type. | constant_keyword | | | -| ecs.version | ECS version | keyword | | | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | | | host.architecture | Operating system architecture. | keyword | | | | host.containerized | If the host is a container. | boolean | | | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | | @@ -478,8 +478,8 @@ An example event for `node` looks as following: | kubernetes.statefulset.name | Kubernetes statefulset name | keyword | | | | orchestrator.cluster.name | Name of the cluster. | keyword | | | | orchestrator.cluster.url | URL of the API used to manage the cluster. | keyword | | | -| service.address | Service address | keyword | | | -| service.type | Service type | keyword | | | +| service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | | ### pod @@ -638,7 +638,7 @@ An example event for `pod` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | | data_stream.type | Data stream type. | constant_keyword | | | -| ecs.version | ECS version | keyword | | | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | | | host.architecture | Operating system architecture. | keyword | | | | host.containerized | If the host is a container. | boolean | | | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | | @@ -688,8 +688,8 @@ An example event for `pod` looks as following: | kubernetes.statefulset.name | Kubernetes statefulset name | keyword | | | | orchestrator.cluster.name | Name of the cluster. | keyword | | | | orchestrator.cluster.url | URL of the API used to manage the cluster. | keyword | | | -| service.address | Service address | keyword | | | -| service.type | Service type | keyword | | | +| service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | | ### system @@ -824,7 +824,7 @@ An example event for `system` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | | data_stream.type | Data stream type. | constant_keyword | | | -| ecs.version | ECS version | keyword | | | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | | | host.architecture | Operating system architecture. | keyword | | | | host.containerized | If the host is a container. | boolean | | | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | | @@ -866,8 +866,8 @@ An example event for `system` looks as following: | kubernetes.system.start_time | Start time | date | | | | orchestrator.cluster.name | Name of the cluster. | keyword | | | | orchestrator.cluster.url | URL of the API used to manage the cluster. | keyword | | | -| service.address | Service address | keyword | | | -| service.type | Service type | keyword | | | +| service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | | ### volume @@ -1000,7 +1000,7 @@ An example event for `volume` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | | data_stream.type | Data stream type. | constant_keyword | | | -| ecs.version | ECS version | keyword | | | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | | | host.architecture | Operating system architecture. | keyword | | | | host.containerized | If the host is a container. | boolean | | | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | | @@ -1041,5 +1041,5 @@ An example event for `volume` looks as following: | kubernetes.volume.name | Volume name | keyword | | | | orchestrator.cluster.name | Name of the cluster. | keyword | | | | orchestrator.cluster.url | URL of the API used to manage the cluster. | keyword | | | -| service.address | Service address | keyword | | | -| service.type | Service type | keyword | | | +| service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | | diff --git a/test/packages/kubernetes/img/metricbeat_kubernetes_overview.png b/test/packages/kubernetes/img/metricbeat_kubernetes_overview.png index 16ff532d27..0b3247dbf8 100644 Binary files a/test/packages/kubernetes/img/metricbeat_kubernetes_overview.png and b/test/packages/kubernetes/img/metricbeat_kubernetes_overview.png differ diff --git a/test/packages/kubernetes/kibana/dashboard/kubernetes-AV4RGUqo5NkDleZmzKuZ.json b/test/packages/kubernetes/kibana/dashboard/kubernetes-AV4RGUqo5NkDleZmzKuZ.json index 0e6477a380..956dd411e2 100644 --- a/test/packages/kubernetes/kibana/dashboard/kubernetes-AV4RGUqo5NkDleZmzKuZ.json +++ b/test/packages/kubernetes/kibana/dashboard/kubernetes-AV4RGUqo5NkDleZmzKuZ.json @@ -19,7 +19,9 @@ }, "panelsJSON": [ { - "embeddableConfig": {}, + "embeddableConfig": { + "enhancements": {} + }, "gridData": { "h": 12, "i": "1", @@ -28,11 +30,14 @@ "y": 0 }, "panelIndex": "1", - "panelRefName": "panel_0", - "version": "7.3.0" + "panelRefName": "panel_1", + "type": "visualization", + "version": "7.14.0" }, { - "embeddableConfig": {}, + "embeddableConfig": { + "enhancements": {} + }, "gridData": { "h": 12, "i": "2", @@ -41,24 +46,30 @@ "y": 24 }, "panelIndex": "2", - "panelRefName": "panel_1", - "version": "7.3.0" + "panelRefName": "panel_2", + "type": "visualization", + "version": "7.14.0" }, { - "embeddableConfig": {}, + "embeddableConfig": { + "enhancements": {} + }, "gridData": { "h": 12, "i": "5", - "w": 12, - "x": 12, + "w": 8, + "x": 16, "y": 0 }, "panelIndex": "5", - "panelRefName": "panel_2", - "version": "7.3.0" + "panelRefName": "panel_5", + "type": "visualization", + "version": "7.14.0" }, { - "embeddableConfig": {}, + "embeddableConfig": { + "enhancements": {} + }, "gridData": { "h": 12, "i": "6", @@ -67,11 +78,14 @@ "y": 12 }, "panelIndex": "6", - "panelRefName": "panel_3", - "version": "7.3.0" + "panelRefName": "panel_6", + "type": "visualization", + "version": "7.14.0" }, { - "embeddableConfig": {}, + "embeddableConfig": { + "enhancements": {} + }, "gridData": { "h": 12, "i": "7", @@ -80,11 +94,14 @@ "y": 24 }, "panelIndex": "7", - "panelRefName": "panel_4", - "version": "7.3.0" + "panelRefName": "panel_7", + "type": "visualization", + "version": "7.14.0" }, { - "embeddableConfig": {}, + "embeddableConfig": { + "enhancements": {} + }, "gridData": { "h": 12, "i": "8", @@ -93,11 +110,14 @@ "y": 36 }, "panelIndex": "8", - "panelRefName": "panel_5", - "version": "7.3.0" + "panelRefName": "panel_8", + "type": "visualization", + "version": "7.14.0" }, { - "embeddableConfig": {}, + "embeddableConfig": { + "enhancements": {} + }, "gridData": { "h": 12, "i": "9", @@ -106,24 +126,30 @@ "y": 36 }, "panelIndex": "9", - "panelRefName": "panel_6", - "version": "7.3.0" + "panelRefName": "panel_9", + "type": "visualization", + "version": "7.14.0" }, { - "embeddableConfig": {}, + "embeddableConfig": { + "enhancements": {} + }, "gridData": { "h": 12, "i": "10", - "w": 12, - "x": 0, + "w": 8, + "x": 8, "y": 0 }, "panelIndex": "10", - "panelRefName": "panel_7", - "version": "7.3.0" + "panelRefName": "panel_10", + "type": "visualization", + "version": "7.14.0" }, { - "embeddableConfig": {}, + "embeddableConfig": { + "enhancements": {} + }, "gridData": { "h": 12, "i": "11", @@ -132,11 +158,14 @@ "y": 48 }, "panelIndex": "11", - "panelRefName": "panel_8", - "version": "7.3.0" + "panelRefName": "panel_11", + "type": "visualization", + "version": "7.14.0" }, { - "embeddableConfig": {}, + "embeddableConfig": { + "enhancements": {} + }, "gridData": { "h": 12, "i": "12", @@ -145,11 +174,14 @@ "y": 48 }, "panelIndex": "12", - "panelRefName": "panel_9", - "version": "7.3.0" + "panelRefName": "panel_12", + "type": "visualization", + "version": "7.14.0" }, { - "embeddableConfig": {}, + "embeddableConfig": { + "enhancements": {} + }, "gridData": { "h": 12, "i": "13", @@ -158,11 +190,14 @@ "y": 12 }, "panelIndex": "13", - "panelRefName": "panel_10", - "version": "7.3.0" + "panelRefName": "panel_13", + "type": "visualization", + "version": "7.14.0" }, { - "embeddableConfig": {}, + "embeddableConfig": { + "enhancements": {} + }, "gridData": { "h": 12, "i": "14", @@ -171,11 +206,14 @@ "y": 12 }, "panelIndex": "14", - "panelRefName": "panel_11", - "version": "7.3.0" + "panelRefName": "panel_14", + "type": "visualization", + "version": "7.14.0" }, { - "embeddableConfig": {}, + "embeddableConfig": { + "enhancements": {} + }, "gridData": { "h": 12, "i": "15", @@ -184,83 +222,144 @@ "y": 12 }, "panelIndex": "15", - "panelRefName": "panel_12", - "version": "7.3.0" + "panelRefName": "panel_15", + "type": "visualization", + "version": "7.14.0" + }, + { + "embeddableConfig": { + "enhancements": {}, + "savedVis": { + "data": { + "aggs": [], + "searchSource": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "description": "", + "params": { + "controls": [ + { + "fieldName": "orchestrator.cluster.name", + "id": "1627653028481", + "indexPatternRefName": "control_19a96b32-b3c9-4aae-a80c-54099f64ffb2_0_index_pattern", + "label": "Cluster Name", + "options": { + "dynamicOptions": true, + "multiselect": true, + "order": "desc", + "size": 5, + "type": "terms" + }, + "parent": "", + "type": "list" + } + ], + "pinFilters": false, + "updateFiltersOnChange": true, + "useTimeFilter": false + }, + "title": "", + "type": "input_control_vis", + "uiState": {} + } + }, + "gridData": { + "h": 12, + "i": "19a96b32-b3c9-4aae-a80c-54099f64ffb2", + "w": 8, + "x": 0, + "y": 0 + }, + "panelIndex": "19a96b32-b3c9-4aae-a80c-54099f64ffb2", + "title": "Cluster Filter [Metrics Kubernetes]", + "type": "visualization", + "version": "7.14.0" } ], "timeRestore": false, "title": "[Metrics Kubernetes] Overview", "version": 1 }, + "coreMigrationVersion": "7.14.0", "id": "kubernetes-AV4RGUqo5NkDleZmzKuZ", "migrationVersion": { - "dashboard": "7.3.0" + "dashboard": "7.14.0" }, "references": [ { "id": "kubernetes-022a54c0-2bf5-11e7-859b-f78b612cde28", - "name": "panel_0", + "name": "1:panel_1", "type": "visualization" }, { "id": "kubernetes-44f12b40-2bf4-11e7-859b-f78b612cde28", - "name": "panel_1", + "name": "2:panel_2", "type": "visualization" }, { "id": "kubernetes-cd059410-2bfb-11e7-859b-f78b612cde28", - "name": "panel_2", + "name": "5:panel_5", "type": "visualization" }, { "id": "kubernetes-e1018b90-2bfb-11e7-859b-f78b612cde28", - "name": "panel_3", + "name": "6:panel_6", "type": "visualization" }, { "id": "kubernetes-d6564360-2bfc-11e7-859b-f78b612cde28", - "name": "panel_4", + "name": "7:panel_7", "type": "visualization" }, { "id": "kubernetes-16fa4470-2bfd-11e7-859b-f78b612cde28", - "name": "panel_5", + "name": "8:panel_8", "type": "visualization" }, { "id": "kubernetes-294546b0-30d6-11e7-8df8-6d3604a72912", - "name": "panel_6", + "name": "9:panel_9", "type": "visualization" }, { "id": "kubernetes-408fccf0-30d6-11e7-8df8-6d3604a72912", - "name": "panel_7", + "name": "10:panel_10", "type": "visualization" }, { "id": "kubernetes-58e644f0-30d6-11e7-8df8-6d3604a72912", - "name": "panel_8", + "name": "11:panel_11", "type": "visualization" }, { "id": "kubernetes-a4c9d360-30df-11e7-8df8-6d3604a72912", - "name": "panel_9", + "name": "12:panel_12", "type": "visualization" }, { "id": "kubernetes-174a6ad0-30e0-11e7-8df8-6d3604a72912", - "name": "panel_10", + "name": "13:panel_13", "type": "visualization" }, { "id": "kubernetes-7aac4fd0-30e0-11e7-8df8-6d3604a72912", - "name": "panel_11", + "name": "14:panel_14", "type": "visualization" }, { "id": "kubernetes-da1ff7c0-30ed-11e7-b9e5-2b5b07213ab3", - "name": "panel_12", + "name": "15:panel_15", "type": "visualization" + }, + { + "id": "metrics-*", + "name": "19a96b32-b3c9-4aae-a80c-54099f64ffb2:control_19a96b32-b3c9-4aae-a80c-54099f64ffb2_0_index_pattern", + "type": "index-pattern" } ], "type": "dashboard" diff --git a/test/packages/kubernetes/kibana/visualization/kubernetes-022a54c0-2bf5-11e7-859b-f78b612cde28.json b/test/packages/kubernetes/kibana/visualization/kubernetes-022a54c0-2bf5-11e7-859b-f78b612cde28.json index f4bbac9730..514663ef52 100644 --- a/test/packages/kubernetes/kibana/visualization/kubernetes-022a54c0-2bf5-11e7-859b-f78b612cde28.json +++ b/test/packages/kubernetes/kibana/visualization/kubernetes-022a54c0-2bf5-11e7-859b-f78b612cde28.json @@ -66,9 +66,10 @@ "type": "metrics" } }, + "coreMigrationVersion": "7.14.0", "id": "kubernetes-022a54c0-2bf5-11e7-859b-f78b612cde28", "migrationVersion": { - "visualization": "7.8.0" + "visualization": "7.14.0" }, "references": [], "type": "visualization" diff --git a/test/packages/kubernetes/kibana/visualization/kubernetes-16fa4470-2bfd-11e7-859b-f78b612cde28.json b/test/packages/kubernetes/kibana/visualization/kubernetes-16fa4470-2bfd-11e7-859b-f78b612cde28.json index 585c28cd1f..25205b8ae0 100644 --- a/test/packages/kubernetes/kibana/visualization/kubernetes-16fa4470-2bfd-11e7-859b-f78b612cde28.json +++ b/test/packages/kubernetes/kibana/visualization/kubernetes-16fa4470-2bfd-11e7-859b-f78b612cde28.json @@ -91,9 +91,10 @@ "type": "metrics" } }, + "coreMigrationVersion": "7.14.0", "id": "kubernetes-16fa4470-2bfd-11e7-859b-f78b612cde28", "migrationVersion": { - "visualization": "7.8.0" + "visualization": "7.14.0" }, "references": [], "type": "visualization" diff --git a/test/packages/kubernetes/kibana/visualization/kubernetes-174a6ad0-30e0-11e7-8df8-6d3604a72912.json b/test/packages/kubernetes/kibana/visualization/kubernetes-174a6ad0-30e0-11e7-8df8-6d3604a72912.json index 1474b18284..d655ac83c7 100644 --- a/test/packages/kubernetes/kibana/visualization/kubernetes-174a6ad0-30e0-11e7-8df8-6d3604a72912.json +++ b/test/packages/kubernetes/kibana/visualization/kubernetes-174a6ad0-30e0-11e7-8df8-6d3604a72912.json @@ -42,6 +42,7 @@ "gauge_max": "", "gauge_style": "half", "gauge_width": "10", + "hide_last_value_indicator": true, "id": "2fe9d3b0-30d5-11e7-8df8-6d3604a72912", "index_pattern": "metrics-*", "interval": "auto", @@ -62,7 +63,6 @@ "type": "sum" } ], - "override_index_pattern": 1, "point_size": 1, "seperate_axis": 0, "series_interval": "10s", @@ -79,9 +79,10 @@ "type": "metrics" } }, + "coreMigrationVersion": "7.14.0", "id": "kubernetes-174a6ad0-30e0-11e7-8df8-6d3604a72912", "migrationVersion": { - "visualization": "7.8.0" + "visualization": "7.14.0" }, "references": [], "type": "visualization" diff --git a/test/packages/kubernetes/kibana/visualization/kubernetes-294546b0-30d6-11e7-8df8-6d3604a72912.json b/test/packages/kubernetes/kibana/visualization/kubernetes-294546b0-30d6-11e7-8df8-6d3604a72912.json index b4e7814492..13f9f2120d 100644 --- a/test/packages/kubernetes/kibana/visualization/kubernetes-294546b0-30d6-11e7-8df8-6d3604a72912.json +++ b/test/packages/kubernetes/kibana/visualization/kubernetes-294546b0-30d6-11e7-8df8-6d3604a72912.json @@ -91,9 +91,10 @@ "type": "metrics" } }, + "coreMigrationVersion": "7.14.0", "id": "kubernetes-294546b0-30d6-11e7-8df8-6d3604a72912", "migrationVersion": { - "visualization": "7.8.0" + "visualization": "7.14.0" }, "references": [], "type": "visualization" diff --git a/test/packages/kubernetes/kibana/visualization/kubernetes-408fccf0-30d6-11e7-8df8-6d3604a72912.json b/test/packages/kubernetes/kibana/visualization/kubernetes-408fccf0-30d6-11e7-8df8-6d3604a72912.json index b66b197e74..a1e624de8b 100644 --- a/test/packages/kubernetes/kibana/visualization/kubernetes-408fccf0-30d6-11e7-8df8-6d3604a72912.json +++ b/test/packages/kubernetes/kibana/visualization/kubernetes-408fccf0-30d6-11e7-8df8-6d3604a72912.json @@ -41,6 +41,7 @@ "gauge_inner_width": 10, "gauge_style": "half", "gauge_width": 10, + "hide_last_value_indicator": true, "id": "4c4690b0-30e0-11e7-8df8-6d3604a72912", "index_pattern": "metrics-*", "interval": "auto", @@ -77,9 +78,10 @@ "type": "metrics" } }, + "coreMigrationVersion": "7.14.0", "id": "kubernetes-408fccf0-30d6-11e7-8df8-6d3604a72912", "migrationVersion": { - "visualization": "7.8.0" + "visualization": "7.14.0" }, "references": [], "type": "visualization" diff --git a/test/packages/kubernetes/kibana/visualization/kubernetes-44f12b40-2bf4-11e7-859b-f78b612cde28.json b/test/packages/kubernetes/kibana/visualization/kubernetes-44f12b40-2bf4-11e7-859b-f78b612cde28.json index 2a9b2fad70..c7d976a72e 100644 --- a/test/packages/kubernetes/kibana/visualization/kubernetes-44f12b40-2bf4-11e7-859b-f78b612cde28.json +++ b/test/packages/kubernetes/kibana/visualization/kubernetes-44f12b40-2bf4-11e7-859b-f78b612cde28.json @@ -146,9 +146,10 @@ "type": "metrics" } }, + "coreMigrationVersion": "7.14.0", "id": "kubernetes-44f12b40-2bf4-11e7-859b-f78b612cde28", "migrationVersion": { - "visualization": "7.8.0" + "visualization": "7.14.0" }, "references": [], "type": "visualization" diff --git a/test/packages/kubernetes/kibana/visualization/kubernetes-58e644f0-30d6-11e7-8df8-6d3604a72912.json b/test/packages/kubernetes/kibana/visualization/kubernetes-58e644f0-30d6-11e7-8df8-6d3604a72912.json index 39a984ada5..c63c528d30 100644 --- a/test/packages/kubernetes/kibana/visualization/kubernetes-58e644f0-30d6-11e7-8df8-6d3604a72912.json +++ b/test/packages/kubernetes/kibana/visualization/kubernetes-58e644f0-30d6-11e7-8df8-6d3604a72912.json @@ -28,6 +28,7 @@ "language": "lucene", "query": "data_stream.dataset:kubernetes.container" }, + "hide_last_value_indicator": true, "id": "5d3692a0-2bfc-11e7-859b-f78b612cde28", "index_pattern": "metrics-*", "interval": "auto", @@ -79,9 +80,10 @@ "type": "metrics" } }, + "coreMigrationVersion": "7.14.0", "id": "kubernetes-58e644f0-30d6-11e7-8df8-6d3604a72912", "migrationVersion": { - "visualization": "7.8.0" + "visualization": "7.14.0" }, "references": [], "type": "visualization" diff --git a/test/packages/kubernetes/kibana/visualization/kubernetes-7aac4fd0-30e0-11e7-8df8-6d3604a72912.json b/test/packages/kubernetes/kibana/visualization/kubernetes-7aac4fd0-30e0-11e7-8df8-6d3604a72912.json index cb3fb9ecbd..442502d4c7 100644 --- a/test/packages/kubernetes/kibana/visualization/kubernetes-7aac4fd0-30e0-11e7-8df8-6d3604a72912.json +++ b/test/packages/kubernetes/kibana/visualization/kubernetes-7aac4fd0-30e0-11e7-8df8-6d3604a72912.json @@ -66,9 +66,10 @@ "type": "metrics" } }, + "coreMigrationVersion": "7.14.0", "id": "kubernetes-7aac4fd0-30e0-11e7-8df8-6d3604a72912", "migrationVersion": { - "visualization": "7.8.0" + "visualization": "7.14.0" }, "references": [], "type": "visualization" diff --git a/test/packages/kubernetes/kibana/visualization/kubernetes-a4c9d360-30df-11e7-8df8-6d3604a72912.json b/test/packages/kubernetes/kibana/visualization/kubernetes-a4c9d360-30df-11e7-8df8-6d3604a72912.json index 931814b3a8..c52c81a96a 100644 --- a/test/packages/kubernetes/kibana/visualization/kubernetes-a4c9d360-30df-11e7-8df8-6d3604a72912.json +++ b/test/packages/kubernetes/kibana/visualization/kubernetes-a4c9d360-30df-11e7-8df8-6d3604a72912.json @@ -27,6 +27,7 @@ "language": "lucene", "query": "data_stream.dataset:kubernetes.container" }, + "hide_last_value_indicator": true, "id": "5d3692a0-2bfc-11e7-859b-f78b612cde28", "index_pattern": "metrics-*", "interval": "auto", @@ -79,9 +80,10 @@ "type": "metrics" } }, + "coreMigrationVersion": "7.14.0", "id": "kubernetes-a4c9d360-30df-11e7-8df8-6d3604a72912", "migrationVersion": { - "visualization": "7.8.0" + "visualization": "7.14.0" }, "references": [], "type": "visualization" diff --git a/test/packages/kubernetes/kibana/visualization/kubernetes-cd059410-2bfb-11e7-859b-f78b612cde28.json b/test/packages/kubernetes/kibana/visualization/kubernetes-cd059410-2bfb-11e7-859b-f78b612cde28.json index 637e7df037..b71985e2b5 100644 --- a/test/packages/kubernetes/kibana/visualization/kubernetes-cd059410-2bfb-11e7-859b-f78b612cde28.json +++ b/test/packages/kubernetes/kibana/visualization/kubernetes-cd059410-2bfb-11e7-859b-f78b612cde28.json @@ -41,6 +41,7 @@ "gauge_inner_width": 10, "gauge_style": "half", "gauge_width": 10, + "hide_last_value_indicator": true, "id": "4c4690b0-30e0-11e7-8df8-6d3604a72912", "index_pattern": "metrics-*", "interval": "auto", @@ -77,9 +78,10 @@ "type": "metrics" } }, + "coreMigrationVersion": "7.14.0", "id": "kubernetes-cd059410-2bfb-11e7-859b-f78b612cde28", "migrationVersion": { - "visualization": "7.8.0" + "visualization": "7.14.0" }, "references": [], "type": "visualization" diff --git a/test/packages/kubernetes/kibana/visualization/kubernetes-d6564360-2bfc-11e7-859b-f78b612cde28.json b/test/packages/kubernetes/kibana/visualization/kubernetes-d6564360-2bfc-11e7-859b-f78b612cde28.json index e848f5227d..0fa2508d74 100644 --- a/test/packages/kubernetes/kibana/visualization/kubernetes-d6564360-2bfc-11e7-859b-f78b612cde28.json +++ b/test/packages/kubernetes/kibana/visualization/kubernetes-d6564360-2bfc-11e7-859b-f78b612cde28.json @@ -126,9 +126,10 @@ "type": "metrics" } }, + "coreMigrationVersion": "7.14.0", "id": "kubernetes-d6564360-2bfc-11e7-859b-f78b612cde28", "migrationVersion": { - "visualization": "7.8.0" + "visualization": "7.14.0" }, "references": [], "type": "visualization" diff --git a/test/packages/kubernetes/kibana/visualization/kubernetes-da1ff7c0-30ed-11e7-b9e5-2b5b07213ab3.json b/test/packages/kubernetes/kibana/visualization/kubernetes-da1ff7c0-30ed-11e7-b9e5-2b5b07213ab3.json index b4a0a9fb89..6cfa254dfa 100644 --- a/test/packages/kubernetes/kibana/visualization/kubernetes-da1ff7c0-30ed-11e7-b9e5-2b5b07213ab3.json +++ b/test/packages/kubernetes/kibana/visualization/kubernetes-da1ff7c0-30ed-11e7-b9e5-2b5b07213ab3.json @@ -42,6 +42,7 @@ "gauge_max": "5", "gauge_style": "half", "gauge_width": "10", + "hide_last_value_indicator": true, "id": "2fe9d3b0-30d5-11e7-8df8-6d3604a72912", "index_pattern": "metrics-*", "interval": "auto", @@ -62,12 +63,9 @@ "type": "sum" } ], - "override_index_pattern": 1, "point_size": 1, "seperate_axis": 0, - "series_index_pattern": "*", "series_interval": "10s", - "series_time_field": "@timestamp", "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none" @@ -81,9 +79,10 @@ "type": "metrics" } }, + "coreMigrationVersion": "7.14.0", "id": "kubernetes-da1ff7c0-30ed-11e7-b9e5-2b5b07213ab3", "migrationVersion": { - "visualization": "7.8.0" + "visualization": "7.14.0" }, "references": [], "type": "visualization" diff --git a/test/packages/kubernetes/kibana/visualization/kubernetes-e1018b90-2bfb-11e7-859b-f78b612cde28.json b/test/packages/kubernetes/kibana/visualization/kubernetes-e1018b90-2bfb-11e7-859b-f78b612cde28.json index 2c3b130132..c99343a8b7 100644 --- a/test/packages/kubernetes/kibana/visualization/kubernetes-e1018b90-2bfb-11e7-859b-f78b612cde28.json +++ b/test/packages/kubernetes/kibana/visualization/kubernetes-e1018b90-2bfb-11e7-859b-f78b612cde28.json @@ -42,6 +42,7 @@ "gauge_max": "5", "gauge_style": "half", "gauge_width": "10", + "hide_last_value_indicator": true, "id": "2fe9d3b0-30d5-11e7-8df8-6d3604a72912", "index_pattern": "metrics-*", "interval": "auto", @@ -62,11 +63,9 @@ "type": "sum" } ], - "override_index_pattern": 1, "point_size": 1, "seperate_axis": 0, "series_interval": "10s", - "series_time_field": "@timestamp", "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none" @@ -80,9 +79,10 @@ "type": "metrics" } }, + "coreMigrationVersion": "7.14.0", "id": "kubernetes-e1018b90-2bfb-11e7-859b-f78b612cde28", "migrationVersion": { - "visualization": "7.8.0" + "visualization": "7.14.0" }, "references": [], "type": "visualization" diff --git a/test/packages/kubernetes/manifest.yml b/test/packages/kubernetes/manifest.yml index 83ec9952ac..7ddd8bd0f9 100644 --- a/test/packages/kubernetes/manifest.yml +++ b/test/packages/kubernetes/manifest.yml @@ -3,14 +3,14 @@ name: kubernetes title: Kubernetes version: 999.999.999 license: basic -description: This Elastic integration collects metrics from Kubernetes clusters +description: Collect logs and metrics from Kubernetes clusters with Elastic Agent. type: integration categories: - containers - kubernetes -release: experimental +release: ga conditions: - kibana.version: "^7.15.0" + kibana.version: "^7.16.0 || ^8.0.0" screenshots: - src: /img/metricbeat_kubernetes_overview.png title: Metricbeat Kubernetes Overview @@ -22,11 +22,48 @@ icons: size: 32x32 type: image/svg+xml policy_templates: + - name: kubelet + title: Kubelet Metrics + description: Collect metrics from Kubernetes Kubelet API with Elastic Agent. + data_streams: + - container + - node + - pod + - system + - volume + inputs: + - type: kubernetes/metrics + title: Collect Kubernetes metrics from Kubelet API + description: Collecting Node, Pod, Container, Volume and System metrics from Kubelet + input_group: metrics + icons: + - src: /img/logo_kubernetes.svg + title: Logo Kubernetes + size: 32x32 + type: image/svg+xml + screenshots: + - src: /img/metricbeat_kubernetes_overview.png + title: Metricbeat Kubernetes Overview + size: 1896x961 + type: image/png - name: kube-state-metrics - title: kube-state-metrics - description: Collect metrics from kube-state-metrics + title: Kube-state-metrics + description: Collect container metrics from Kubernetes Kube-state-metrics with Elastic Agent. data_streams: + - state_container + - state_cronjob + - state_daemonset + - state_deployment + - state_job + - state_node + - state_persistentvolume + - state_persistentvolumeclaim - state_pod + - state_replicaset + - state_resourcequota + - state_service + - state_statefulset + - state_storageclass inputs: - type: kubernetes/metrics title: Collect Kubernetes metrics from kube-state-metrics @@ -42,5 +79,124 @@ policy_templates: title: Metricbeat Kubernetes Overview size: 1896x961 type: image/png + - name: kube-apiserver + title: Kube-apiserver Metrics + description: Collect metrics from Kubernetes API Server with Elastic Agent. + data_streams: + - apiserver + inputs: + - type: kubernetes/metrics + title: Collect Kubernetes metrics from Kubernetes API Server + description: Collecting metrics from Kubernetes API Server + input_group: metrics + icons: + - src: /img/logo_kubernetes.svg + title: Logo Kubernetes + size: 32x32 + type: image/svg+xml + screenshots: + - src: /img/metricbeat_kubernetes_overview.png + title: Metricbeat Kubernetes Overview + size: 1896x961 + type: image/png + - name: kube-proxy + title: Kube-proxy Metrics + description: Collect metrics from Kubernetes Proxy server with Elastic Agent. + data_streams: + - proxy + inputs: + - type: kubernetes/metrics + title: Collect Kubernetes metrics from Kubernetes Proxy + description: Collecting metrics from Kubernetes Proxy + input_group: metrics + icons: + - src: /img/logo_kubernetes.svg + title: Logo Kubernetes + size: 32x32 + type: image/svg+xml + screenshots: + - src: /img/metricbeat-kubernetes-proxy.png + title: Metricbeat Kubernetes Proxy + size: 1854x920 + type: image/png + - name: kube-scheduler + title: Kube-scheduler Metrics + description: Collect metrics from Kubernetes Scheduler with Elastic Agent. + data_streams: + - scheduler + inputs: + - type: kubernetes/metrics + title: Collect Kubernetes metrics from Kubernetes Scheduler + description: Collecting metrics from Kubernetes Scheduler + input_group: metrics + icons: + - src: /img/logo_kubernetes.svg + title: Logo Kubernetes + size: 32x32 + type: image/svg+xml + screenshots: + - src: /img/metricbeat_kubernetes_scheduler.png + title: Metricbeat Kubernetes Scheduler + size: 1856x897 + type: image/png + - name: kube-controller-manager + title: Kube-controller-manager Metrics + description: Collect metrics from Kubernetes Controller Manager with Elastic Agent. + data_streams: + - controllermanager + inputs: + - type: kubernetes/metrics + title: Collect Kubernetes metrics from Kubernetes controller-manager + description: Collecting metrics from Kubernetes controller-manager + input_group: metrics + icons: + - src: /img/logo_kubernetes.svg + title: Logo Kubernetes + size: 32x32 + type: image/svg+xml + screenshots: + - src: /img/metricbeat-kubernetes-controllermanager.png + title: Metricbeat Kubernetes Controller Manager + size: 1896x961 + type: image/png + - name: events + title: Kubernetes Event Metrics + description: Collect events related metrics from Kubernetes API server with Elastic Agent. + data_streams: + - event + inputs: + - type: kubernetes/metrics + title: Collect Kubernetes events from Kubernetes API Server + description: Collecting events from Kubernetes API Server + input_group: metrics + icons: + - src: /img/logo_kubernetes.svg + title: Logo Kubernetes + size: 32x32 + type: image/svg+xml + screenshots: + - src: /img/metricbeat_kubernetes_overview.png + title: Metricbeat Kubernetes Overview + size: 1896x961 + type: image/png + - name: container-logs + title: Kubernetes Container Logs + description: Collect container related logs from Kubernetes clusters with Elastic Agent. + data_streams: + - container_logs + inputs: + - type: filestream + title: Collect Kubernetes container logs + description: Collect Kubernetes container logs + icons: + - src: /img/logo_kubernetes.svg + title: Logo Kubernetes + size: 32x32 + type: image/svg+xml + screenshots: + - src: /img/metricbeat_kubernetes_overview.png + title: Metricbeat Kubernetes Overview + size: 1896x961 + type: image/png owner: github: elastic/integrations diff --git a/test/packages/log/changelog.yml b/test/packages/log/changelog.yml index 1a3f8f8a68..0561af4746 100644 --- a/test/packages/log/changelog.yml +++ b/test/packages/log/changelog.yml @@ -1,6 +1,21 @@ # newer versions go on top +- version: "999.999.999" + changes: + - description: Uniform with guidelines + type: enhancement + link: https://github.com/elastic/integrations/pull/2059 +- version: "0.5.0" + changes: + - description: Update integration description + type: enhancement + link: https://github.com/elastic/integrations/pull/1364 - version: "0.4.6" + changes: + - description: Updating package owner + type: enhancement + link: https://github.com/elastic/integrations/pull/766 +- version: "0.1.0" changes: - description: initial release type: enhancement # can be one of: enhancement, bugfix, breaking-change - link: https://github.com/elastic/elastic-package/pull/224 + link: https://github.com/elastic/package-storage/pull/30 diff --git a/test/packages/log/manifest.yml b/test/packages/log/manifest.yml index 659d38071a..a53742e70f 100644 --- a/test/packages/log/manifest.yml +++ b/test/packages/log/manifest.yml @@ -1,11 +1,10 @@ format_version: 1.0.0 name: log -title: Custom logs -description: > - Collect your custom logs. - +title: Custom Logs +description: >- + Collect custom logs with Elastic Agent. type: integration -version: 0.4.6 +version: 999.999.999 release: experimental license: basic categories: @@ -22,4 +21,4 @@ icons: - src: "/img/icon.svg" type: "image/svg+xml" owner: - github: elastic/integrations-services + github: elastic/integrations diff --git a/test/packages/nginx/_dev/build/build.yml b/test/packages/nginx/_dev/build/build.yml index 149139dc2a..08d85edcf9 100644 --- a/test/packages/nginx/_dev/build/build.yml +++ b/test/packages/nginx/_dev/build/build.yml @@ -1,3 +1,3 @@ dependencies: ecs: - reference: git@0b8b7d6121340e99a1eb463c91fd1bc7c9eb2e41 + reference: git@1.12 diff --git a/test/packages/nginx/_dev/build/docs/README.md b/test/packages/nginx/_dev/build/docs/README.md index a4fcb130a7..31cb9d66f6 100644 --- a/test/packages/nginx/_dev/build/docs/README.md +++ b/test/packages/nginx/_dev/build/docs/README.md @@ -1,7 +1,61 @@ # Nginx Integration +This integration periodically fetches metrics from [Nginx](https://nginx.org/) servers. It can parse access and error +logs created by the HTTP server. + +## Compatibility + +The Nginx `stubstatus` metrics was tested with Nginx 1.19.5 and are expected to work with all version >= 1.9. +The logs were tested with version 1.19.5. +On Windows, the module was tested with Nginx installed from the Chocolatey repository. + +## Logs + +**Timezone support** + +This datasource parses logs that don’t contain timezone information. For these logs, the Elastic Agent reads the local +timezone and uses it when parsing to convert the timestamp to UTC. The timezone to be used for parsing is included +in the event in the `event.timezone` field. + +To disable this conversion, the event.timezone field can be removed with the drop_fields processor. + +If logs are originated from systems or applications with a different timezone to the local one, the `event.timezone` +field can be overwritten with the original timezone using the add_fields processor. + +### Access Logs + +Access logs collects the nginx access logs. + +{{event "access"}} + {{fields "access"}} +### Error Logs + +Error logs collects the nginx error logs. + +{{event "error"}} + {{fields "error"}} -{{fields "stubstatus"}} \ No newline at end of file +## Metrics + +### Stub Status Metrics + +The Nginx `stubstatus` stream collects data from the Nginx `ngx_http_stub_status` module. It scrapes the server status +data from the web page generated by `ngx_http_stub_status`. Please verify that your Nginx distribution comes with the mentioned +module and it's enabled in the Nginx configuration file: + +``` +location /nginx_status { + stub_status; + allow 127.0.0.1; # only allow requests from localhost + deny all; # deny all other hosts +} +``` + +It's highly recommended to replace `127.0.0.1` with your server’s IP address and make sure that this page accessible to only you. + +{{event "stubstatus"}} + +{{fields "stubstatus"}} diff --git a/test/packages/nginx/_dev/deploy/docker/Dockerfile b/test/packages/nginx/_dev/deploy/docker/Dockerfile new file mode 100755 index 0000000000..9ab90099e0 --- /dev/null +++ b/test/packages/nginx/_dev/deploy/docker/Dockerfile @@ -0,0 +1,6 @@ +ARG SERVICE_VERSION=${SERVICE_VERSION:-1.19.5} +FROM nginx:${SERVICE_VERSION} +RUN sed -i "/jessie-updates/d" /etc/apt/sources.list +RUN apt-get update && apt-get install -y curl +HEALTHCHECK --interval=1s --retries=90 CMD curl -f http://localhost/server-status +COPY ./nginx.conf /etc/nginx/ diff --git a/test/packages/nginx/_dev/deploy/docker/docker-compose.yml b/test/packages/nginx/_dev/deploy/docker/docker-compose.yml new file mode 100644 index 0000000000..ed32af8b43 --- /dev/null +++ b/test/packages/nginx/_dev/deploy/docker/docker-compose.yml @@ -0,0 +1,8 @@ +version: '2.3' +services: + nginx: + build: . + ports: + - 80 + volumes: + - ${SERVICE_LOGS_DIR}:/var/log/nginx diff --git a/test/packages/nginx/_dev/deploy/docker/nginx.conf b/test/packages/nginx/_dev/deploy/docker/nginx.conf new file mode 100755 index 0000000000..f530f31a64 --- /dev/null +++ b/test/packages/nginx/_dev/deploy/docker/nginx.conf @@ -0,0 +1,32 @@ +user nginx; +worker_processes 1; + +error_log /dev/stderr warn; +pid /var/run/nginx.pid; + + +events { + worker_connections 1024; +} + + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + access_log /var/log/nginx/access.log main; + + server { + listen 80; + server_name localhost; + + location /server-status { + stub_status on; + } + } + + include /etc/nginx/conf.d/*; +} diff --git a/test/packages/nginx/_dev/deploy/variants.yml b/test/packages/nginx/_dev/deploy/variants.yml new file mode 100644 index 0000000000..a5d765e577 --- /dev/null +++ b/test/packages/nginx/_dev/deploy/variants.yml @@ -0,0 +1,4 @@ +variants: + v1: + SERVICE_VERSION: 1.19.5 +default: v1 diff --git a/test/packages/nginx/changelog.yml b/test/packages/nginx/changelog.yml index f02015184f..e131c35508 100644 --- a/test/packages/nginx/changelog.yml +++ b/test/packages/nginx/changelog.yml @@ -1,6 +1,98 @@ # newer versions go on top - version: "999.999.999" + changes: + - description: Change test public IPs to the supported subset + type: bugfix + link: https://github.com/elastic/integrations/pull/2327 +- version: "1.2.1" + changes: + - description: Fix ML module manifest query to ignore frozen and cold tiers + type: bugfix + link: https://github.com/elastic/integrations/pull/2219 +- version: "1.2.0" + changes: + - description: Release nginx package for v8.0.0 + type: enhancement + link: | + https://github.com/elastic/integrations/pull/2176 +- version: "1.1.2" + changes: + - description: Uniform with guidelines + type: enhancement + link: | + https://github.com/elastic/integrations/pull/2052 +- version: "1.1.1" + changes: + - description: Fix logic that checks for the 'forwarded' tag + type: bugfix + link: https://github.com/elastic/integrations/pull/1835 +- version: "1.1.0" + changes: + - description: Update to ECS 1.12.0 + type: enhancement + link: https://github.com/elastic/integrations/pull/1701 +- version: "1.0.0" + changes: + - description: Release Nginx as GA + type: enhancement + link: https://github.com/elastic/integrations/pull/1614 +- version: "0.8.2" + changes: + - description: Convert to generated ECS fields + type: enhancement + link: https://github.com/elastic/integrations/pull/1491 +- version: "0.8.1" + changes: + - description: update to ECS 1.11.0 + type: enhancement + link: https://github.com/elastic/integrations/pull/1398 +- version: "0.8.0" + changes: + - description: Update integration description + type: enhancement + link: https://github.com/elastic/integrations/pull/1364 +- version: "0.7.0" + changes: + - description: Set "event.module" and "event.dataset" + type: enhancement + link: https://github.com/elastic/integrations/pull/1234 +- version: "0.6.2" + changes: + - description: Add support for Splunk authorization tokens + type: enhancement + link: https://github.com/elastic/integrations/pull/1147 +- version: "0.6.1" + changes: + - description: Fix bug in Third Party REST API ingest pipeline + type: bugfix + link: https://github.com/elastic/integrations/pull/1201 +- version: "0.6.0" + changes: + - description: update to ECS 1.10.0 and add event.original options + type: enhancement + link: https://github.com/elastic/integrations/pull/1065 +- version: "0.5.0" + changes: + - description: Adds ML jobs for finding unusual activity in HTTP access logs + type: enhancement + link: https://github.com/elastic/integrations/pull/912 +- version: "0.4.1" + changes: + - description: update to ECS 1.9.0 + type: enhancement + link: https://github.com/elastic/integrations/pull/859 +- version: "0.3.11" + changes: + - description: Updating package owner + type: enhancement + link: https://github.com/elastic/integrations/pull/766 +- version: "0.3.10" + changes: + - description: Fix compatibility with Kibana + type: enhancement # can be one of: enhancement, bugfix, breaking-change + link: https://github.com/elastic/integrations/pull/740 +- version: "0.1.0" changes: - description: initial release type: enhancement # can be one of: enhancement, bugfix, breaking-change - link: https://github.com/elastic/elastic-package/pull/109 + link: https://github.com/elastic/integrations/pull/21 diff --git a/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-access-event.json b/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-access-event.json deleted file mode 100644 index 13af1dc3fc..0000000000 --- a/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-access-event.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "events": [ - { - "@timestamp": "2016-10-25T12:49:34.000Z", - "message": "127.0.0.1 - - [07/Dec/2016:11:04:37 +0100] \"GET /test HTTP/1.1\" 404 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"\n" - }, - { - "@timestamp": "2016-10-25T12:49:34.000Z", - "message": "127.0.0.1 - - [07/Dec/2016:11:05:07 +0100] \"GET /test HTTP/1.1\" 404 169 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"\n" - } - ] -} \ No newline at end of file diff --git a/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-access-event.json-config.yml b/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-access-event.json-config.yml deleted file mode 100644 index 66b4a2670f..0000000000 --- a/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-access-event.json-config.yml +++ /dev/null @@ -1,4 +0,0 @@ -fields: - event.custom_property: "foobar" -dynamic_fields: - url.original: "^/.*$" diff --git a/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-access-event.json-expected.json b/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-access-event.json-expected.json deleted file mode 100644 index 77fbaf7f6a..0000000000 --- a/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-access-event.json-expected.json +++ /dev/null @@ -1,122 +0,0 @@ -{ - "expected": [ - { - "@timestamp": "2016-12-07T10:04:37.000Z", - "nginx": { - "access": { - "remote_ip_list": [ - "127.0.0.1" - ] - } - }, - "related": { - "ip": [ - "127.0.0.1" - ] - }, - "http": { - "request": { - "method": "get" - }, - "version": "1.1", - "response": { - "body": { - "bytes": 571 - }, - "status_code": 404 - } - }, - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, - "event": { - "custom_property": "foobar", - "category": [ - "web" - ], - "type": [ - "access" - ], - "created": "2016-10-25T12:49:34.000Z", - "kind": "event", - "outcome": "failure" - }, - "user_agent": { - "name": "Chrome", - "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36", - "os": { - "name": "Mac OS X", - "version": "10.12.0", - "full": "Mac OS X 10.12.0" - }, - "device": { - "name": "Mac" - }, - "version": "54.0.2840.98" - }, - "url": { - "original": "/test" - } - }, - { - "@timestamp": "2016-12-07T10:05:07.000Z", - "nginx": { - "access": { - "remote_ip_list": [ - "127.0.0.1" - ] - } - }, - "related": { - "ip": [ - "127.0.0.1" - ] - }, - "http": { - "request": { - "method": "get" - }, - "version": "1.1", - "response": { - "body": { - "bytes": 169 - }, - "status_code": 404 - } - }, - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, - "event": { - "custom_property": "foobar", - "category": [ - "web" - ], - "type": [ - "access" - ], - "created": "2016-10-25T12:49:34.000Z", - "kind": "event", - "outcome": "failure" - }, - "user_agent": { - "name": "Firefox", - "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0", - "os": { - "name": "Mac OS X", - "version": "10.12", - "full": "Mac OS X 10.12" - }, - "device": { - "name": "Mac" - }, - "version": "49.0." - }, - "url": { - "original": "/test" - } - } - ] -} \ No newline at end of file diff --git a/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-access-raw.log b/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-access-raw.log deleted file mode 100644 index 3559803565..0000000000 --- a/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-access-raw.log +++ /dev/null @@ -1,12 +0,0 @@ -127.0.0.1 - - [07/Dec/2016:11:04:37 +0100] "GET /test1 HTTP/1.1" 404 571 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36" -127.0.0.1 - - [07/Dec/2016:11:04:58 +0100] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0" -127.0.0.1 - - [07/Dec/2016:11:04:59 +0100] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0" -127.0.0.1 - - [07/Dec/2016:11:05:07 +0100] "GET /taga HTTP/1.1" 404 169 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0" -77.179.66.156 - - [07/Dec/2016:10:34:43 +0100] "GET / HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36" -77.179.66.156 - - [07/Dec/2016:10:34:43 +0100] "GET /favicon.ico HTTP/1.1" 404 571 "http://localhost:8080/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36" -77.179.66.156 - - [07/Dec/2016:10:43:18 +0100] "GET /test HTTP/1.1" 404 571 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36" -77.179.66.156 - - [07/Dec/2016:10:43:21 +0100] "GET /test HTTP/1.1" 404 571 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36" -77.179.66.156 - - [07/Dec/2016:10:43:23 +0100] "GET /test1 HTTP/1.1" 404 571 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36" -77.179.66.156 - - [25/Oct/2016:14:49:33 +0200] "GET / HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36" -77.179.66.156 - - [25/Oct/2016:14:49:34 +0200] "GET /favicon.ico HTTP/1.1" 404 571 "http://localhost:8080/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36" -77.179.66.156 - - [25/Oct/2016:14:50:44 +0200] "GET /adsasd HTTP/1.1" 404 571 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36" \ No newline at end of file diff --git a/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-access-single.log b/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-access-single.log deleted file mode 100644 index 3991526a4a..0000000000 --- a/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-access-single.log +++ /dev/null @@ -1 +0,0 @@ -127.0.0.1 - - [07/Dec/2016:11:04:37 +0100] "GET /test1 HTTP/1.1" 404 571 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36" \ No newline at end of file diff --git a/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-access-single.log-config.yml b/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-access-single.log-config.yml deleted file mode 100644 index 958d74a23e..0000000000 --- a/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-access-single.log-config.yml +++ /dev/null @@ -1,4 +0,0 @@ -multiline: - first_line_pattern: "^(?:[0-9]{1,3}\\.){3}[0-9]{1,3}" -fields: - "@timestamp": "2020-04-28T11:07:58.223Z" diff --git a/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-access-single.log-expected.json b/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-access-single.log-expected.json deleted file mode 100644 index d253aba78b..0000000000 --- a/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-access-single.log-expected.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "expected": [ - { - "@timestamp": "2016-12-07T10:04:37.000Z", - "nginx": { - "access": { - "remote_ip_list": [ - "127.0.0.1" - ] - } - }, - "related": { - "ip": [ - "127.0.0.1" - ] - }, - "http": { - "request": { - "method": "get" - }, - "version": "1.1", - "response": { - "body": { - "bytes": 571 - }, - "status_code": 404 - } - }, - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, - "event": { - "category": [ - "web" - ], - "type": [ - "access" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event", - "outcome": "failure" - }, - "user_agent": { - "name": "Chrome", - "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36", - "os": { - "name": "Mac OS X", - "version": "10.12.0", - "full": "Mac OS X 10.12.0" - }, - "device": { - "name": "Mac" - }, - "version": "54.0.2840.98" - }, - "url": { - "original": "/test1" - } - } - ] -} \ No newline at end of file diff --git a/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-access.log b/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-access.log new file mode 100644 index 0000000000..10715a8ff9 --- /dev/null +++ b/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-access.log @@ -0,0 +1,14 @@ +67.43.156.13 - - [25/Oct/2016:14:49:33 +0200] "GET / HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36" +67.43.156.13 - - [25/Oct/2016:14:49:34 +0200] "GET /favicon.ico HTTP/1.1" 404 571 "http://localhost:8080/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36" +67.43.156.13 - - [25/Oct/2016:14:50:44 +0200] "GET /adsasd HTTP/1.1" 404 571 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36" +67.43.156.13 - - [07/Dec/2016:10:34:43 +0100] "GET / HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36" +67.43.156.13 - - [07/Dec/2016:10:34:43 +0100] "GET /favicon.ico HTTP/1.1" 404 571 "http://localhost:8080/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36" +67.43.156.13 - - [07/Dec/2016:10:43:18 +0100] "GET /test HTTP/1.1" 404 571 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36" +67.43.156.13 - - [07/Dec/2016:10:43:21 +0100] "GET /test HTTP/1.1" 404 571 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36" +67.43.156.13 - - [07/Dec/2016:10:43:23 +0100] "GET /test1 HTTP/1.1" 404 571 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36" +127.0.0.1 - - [07/Dec/2016:11:04:37 +0100] "GET /test1 HTTP/1.1" 404 571 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36" +127.0.0.1 - - [07/Dec/2016:11:04:58 +0100] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0" +127.0.0.1 - - [07/Dec/2016:11:04:59 +0100] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0" +127.0.0.1 - - [07/Dec/2016:11:05:07 +0100] "GET /taga HTTP/1.1" 404 169 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0" +lessons.example.com 192.168.0.1 - - [09/Jun/2020:12:10:39 -0700] "GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1" 206 7648063 "http://lessons.example.com/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4" "Mozilla/5.0 (Linux; Android 5.1.1; KFFOWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/81.2.16 like Chrome/81.0.4044.138 Safari/537.36" +lessons.example.com 192.168.0.1 - - [09/Jun/2020:12:15:39 -0700] "GET /%D0%A0%D1%83%D1%81%D1%81%D0%BA%D0%B0%D1%8F%20%D1%88%D0%BA%D0%BE%D0%BB%D0%B0%20-%20InternetUrok%201%D0%BA%D0%BB%D0%B0%D1%81%D1%81/ HTTP/1.1" 206 7648063 "http://lessons.example.com/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4" "Mozilla/5.0 (Linux; Android 5.1.1; KFFOWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/81.2.16 like Chrome/81.0.4044.138 Safari/537.36" diff --git a/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-access-raw.log-expected.json b/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-access.log-expected.json similarity index 70% rename from test/packages/nginx/data_stream/access/_dev/test/pipeline/test-access-raw.log-expected.json rename to test/packages/nginx/data_stream/access/_dev/test/pipeline/test-access.log-expected.json index cea464fa99..778ef7889f 100644 --- a/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-access-raw.log-expected.json +++ b/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-access.log-expected.json @@ -1,17 +1,32 @@ { "expected": [ { - "@timestamp": "2016-12-07T10:04:37.000Z", "nginx": { "access": { "remote_ip_list": [ - "127.0.0.1" + "67.43.156.13" ] } }, + "source": { + "address": "67.43.156.13", + "ip": "67.43.156.13" + }, + "url": { + "path": "/", + "original": "/" + }, + "tags": [ + "preserve_original_event" + ], + "@timestamp": "2016-10-25T12:49:33.000Z", + "ecs": { + "version": "1.12.0" + }, + "_tmp": {}, "related": { "ip": [ - "127.0.0.1" + "67.43.156.13" ] }, "http": { @@ -21,29 +36,27 @@ "version": "1.1", "response": { "body": { - "bytes": 571 + "bytes": 612 }, - "status_code": 404 + "status_code": 200 } }, - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, "event": { + "ingested": "2021-12-09T13:41:40.653051900Z", + "original": "67.43.156.13 - - [25/Oct/2016:14:49:33 +0200] \"GET / HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36\"", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", "category": [ "web" ], "type": [ "access" ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event", - "outcome": "failure" + "outcome": "success" }, "user_agent": { "name": "Chrome", - "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36", + "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36", "os": { "name": "Mac OS X", "version": "10.12.0", @@ -52,82 +65,106 @@ "device": { "name": "Mac" }, - "version": "54.0.2840.98" - }, - "url": { - "original": "/test1" + "version": "54.0.2840.59" } }, { - "@timestamp": "2016-12-07T10:04:58.000Z", "nginx": { "access": { "remote_ip_list": [ - "127.0.0.1" + "67.43.156.13" ] } }, + "source": { + "address": "67.43.156.13", + "ip": "67.43.156.13" + }, + "url": { + "path": "/favicon.ico", + "extension": "ico", + "original": "/favicon.ico" + }, + "tags": [ + "preserve_original_event" + ], + "@timestamp": "2016-10-25T12:49:34.000Z", + "ecs": { + "version": "1.12.0" + }, + "_tmp": {}, "related": { "ip": [ - "127.0.0.1" + "67.43.156.13" ] }, "http": { "request": { - "method": "get" + "method": "get", + "referrer": "http://localhost:8080/" }, "version": "1.1", "response": { "body": { - "bytes": 0 + "bytes": 571 }, - "status_code": 304 + "status_code": 404 } }, - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, "event": { + "ingested": "2021-12-09T13:41:40.653067Z", + "original": "67.43.156.13 - - [25/Oct/2016:14:49:34 +0200] \"GET /favicon.ico HTTP/1.1\" 404 571 \"http://localhost:8080/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36\"", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", "category": [ "web" ], "type": [ "access" ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event", - "outcome": "success" + "outcome": "failure" }, "user_agent": { - "name": "Firefox", - "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0", + "name": "Chrome", + "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36", "os": { "name": "Mac OS X", - "version": "10.12", - "full": "Mac OS X 10.12" + "version": "10.12.0", + "full": "Mac OS X 10.12.0" }, "device": { "name": "Mac" }, - "version": "49.0." - }, - "url": { - "original": "/" + "version": "54.0.2840.59" } }, { - "@timestamp": "2016-12-07T10:04:59.000Z", "nginx": { "access": { "remote_ip_list": [ - "127.0.0.1" + "67.43.156.13" ] } }, + "source": { + "address": "67.43.156.13", + "ip": "67.43.156.13" + }, + "url": { + "path": "/adsasd", + "original": "/adsasd" + }, + "tags": [ + "preserve_original_event" + ], + "@timestamp": "2016-10-25T12:50:44.000Z", + "ecs": { + "version": "1.12.0" + }, + "_tmp": {}, "related": { "ip": [ - "127.0.0.1" + "67.43.156.13" ] }, "http": { @@ -137,55 +174,65 @@ "version": "1.1", "response": { "body": { - "bytes": 0 + "bytes": 571 }, - "status_code": 304 + "status_code": 404 } }, - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, "event": { + "ingested": "2021-12-09T13:41:40.653072600Z", + "original": "67.43.156.13 - - [25/Oct/2016:14:50:44 +0200] \"GET /adsasd HTTP/1.1\" 404 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36\"", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", "category": [ "web" ], "type": [ "access" ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event", - "outcome": "success" + "outcome": "failure" }, "user_agent": { - "name": "Firefox", - "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0", + "name": "Chrome", + "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36", "os": { "name": "Mac OS X", - "version": "10.12", - "full": "Mac OS X 10.12" + "version": "10.12.0", + "full": "Mac OS X 10.12.0" }, "device": { "name": "Mac" }, - "version": "49.0." - }, - "url": { - "original": "/" + "version": "54.0.2840.59" } }, { - "@timestamp": "2016-12-07T10:05:07.000Z", "nginx": { "access": { "remote_ip_list": [ - "127.0.0.1" + "67.43.156.13" ] } }, + "source": { + "address": "67.43.156.13", + "ip": "67.43.156.13" + }, + "url": { + "path": "/", + "original": "/" + }, + "tags": [ + "preserve_original_event" + ], + "@timestamp": "2016-12-07T09:34:43.000Z", + "ecs": { + "version": "1.12.0" + }, + "_tmp": {}, "related": { "ip": [ - "127.0.0.1" + "67.43.156.13" ] }, "http": { @@ -195,101 +242,93 @@ "version": "1.1", "response": { "body": { - "bytes": 169 + "bytes": 612 }, - "status_code": 404 + "status_code": 200 } }, - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, "event": { + "ingested": "2021-12-09T13:41:40.653079800Z", + "original": "67.43.156.13 - - [07/Dec/2016:10:34:43 +0100] \"GET / HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", "category": [ "web" ], "type": [ "access" ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event", - "outcome": "failure" + "outcome": "success" }, "user_agent": { - "name": "Firefox", - "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0", + "name": "Chrome", + "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36", "os": { "name": "Mac OS X", - "version": "10.12", - "full": "Mac OS X 10.12" + "version": "10.12.0", + "full": "Mac OS X 10.12.0" }, "device": { "name": "Mac" }, - "version": "49.0." - }, - "url": { - "original": "/taga" + "version": "54.0.2840.98" } }, { - "@timestamp": "2016-12-07T09:34:43.000Z", "nginx": { "access": { "remote_ip_list": [ - "77.179.66.156" + "67.43.156.13" ] } }, + "source": { + "address": "67.43.156.13", + "ip": "67.43.156.13" + }, + "url": { + "path": "/favicon.ico", + "extension": "ico", + "original": "/favicon.ico" + }, + "tags": [ + "preserve_original_event" + ], + "@timestamp": "2016-12-07T09:34:43.000Z", + "ecs": { + "version": "1.12.0" + }, + "_tmp": {}, "related": { "ip": [ - "77.179.66.156" + "67.43.156.13" ] }, "http": { "request": { - "method": "get" + "method": "get", + "referrer": "http://localhost:8080/" }, "version": "1.1", "response": { "body": { - "bytes": 612 + "bytes": 571 }, - "status_code": 200 + "status_code": 404 } }, - "source": { - "geo": { - "continent_name": "Europe", - "region_iso_code": "DE-RP", - "city_name": "Germersheim", - "country_iso_code": "DE", - "country_name": "Germany", - "region_name": "Rheinland-Pfalz", - "location": { - "lon": 8.3639, - "lat": 49.2231 - } - }, - "as": { - "number": 6805, - "organization": { - "name": "Telefonica Germany" - } - }, - "address": "77.179.66.156", - "ip": "77.179.66.156" - }, "event": { + "ingested": "2021-12-09T13:41:40.653088800Z", + "original": "67.43.156.13 - - [07/Dec/2016:10:34:43 +0100] \"GET /favicon.ico HTTP/1.1\" 404 571 \"http://localhost:8080/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", "category": [ "web" ], "type": [ "access" ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event", - "outcome": "success" + "outcome": "failure" }, "user_agent": { "name": "Chrome", @@ -303,29 +342,40 @@ "name": "Mac" }, "version": "54.0.2840.98" - }, - "url": { - "original": "/" } }, { - "@timestamp": "2016-12-07T09:34:43.000Z", "nginx": { "access": { "remote_ip_list": [ - "77.179.66.156" + "67.43.156.13" ] } }, + "source": { + "address": "67.43.156.13", + "ip": "67.43.156.13" + }, + "url": { + "path": "/test", + "original": "/test" + }, + "tags": [ + "preserve_original_event" + ], + "@timestamp": "2016-12-07T09:43:18.000Z", + "ecs": { + "version": "1.12.0" + }, + "_tmp": {}, "related": { "ip": [ - "77.179.66.156" + "67.43.156.13" ] }, "http": { "request": { - "method": "get", - "referrer": "http://localhost:8080/" + "method": "get" }, "version": "1.1", "response": { @@ -335,37 +385,17 @@ "status_code": 404 } }, - "source": { - "geo": { - "continent_name": "Europe", - "region_iso_code": "DE-RP", - "city_name": "Germersheim", - "country_iso_code": "DE", - "country_name": "Germany", - "region_name": "Rheinland-Pfalz", - "location": { - "lon": 8.3639, - "lat": 49.2231 - } - }, - "as": { - "number": 6805, - "organization": { - "name": "Telefonica Germany" - } - }, - "address": "77.179.66.156", - "ip": "77.179.66.156" - }, "event": { + "ingested": "2021-12-09T13:41:40.653098400Z", + "original": "67.43.156.13 - - [07/Dec/2016:10:43:18 +0100] \"GET /test HTTP/1.1\" 404 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", "category": [ "web" ], "type": [ "access" ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event", "outcome": "failure" }, "user_agent": { @@ -380,23 +410,35 @@ "name": "Mac" }, "version": "54.0.2840.98" - }, - "url": { - "original": "/favicon.ico" } }, { - "@timestamp": "2016-12-07T09:43:18.000Z", "nginx": { "access": { "remote_ip_list": [ - "77.179.66.156" + "67.43.156.13" ] } }, + "source": { + "address": "67.43.156.13", + "ip": "67.43.156.13" + }, + "url": { + "path": "/test", + "original": "/test" + }, + "tags": [ + "preserve_original_event" + ], + "@timestamp": "2016-12-07T09:43:21.000Z", + "ecs": { + "version": "1.12.0" + }, + "_tmp": {}, "related": { "ip": [ - "77.179.66.156" + "67.43.156.13" ] }, "http": { @@ -411,37 +453,17 @@ "status_code": 404 } }, - "source": { - "geo": { - "continent_name": "Europe", - "region_iso_code": "DE-RP", - "city_name": "Germersheim", - "country_iso_code": "DE", - "country_name": "Germany", - "region_name": "Rheinland-Pfalz", - "location": { - "lon": 8.3639, - "lat": 49.2231 - } - }, - "as": { - "number": 6805, - "organization": { - "name": "Telefonica Germany" - } - }, - "address": "77.179.66.156", - "ip": "77.179.66.156" - }, "event": { + "ingested": "2021-12-09T13:41:40.653105800Z", + "original": "67.43.156.13 - - [07/Dec/2016:10:43:21 +0100] \"GET /test HTTP/1.1\" 404 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", "category": [ "web" ], "type": [ "access" ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event", "outcome": "failure" }, "user_agent": { @@ -456,23 +478,35 @@ "name": "Mac" }, "version": "54.0.2840.98" - }, - "url": { - "original": "/test" } }, { - "@timestamp": "2016-12-07T09:43:21.000Z", "nginx": { "access": { "remote_ip_list": [ - "77.179.66.156" + "67.43.156.13" ] } }, + "source": { + "address": "67.43.156.13", + "ip": "67.43.156.13" + }, + "url": { + "path": "/test1", + "original": "/test1" + }, + "tags": [ + "preserve_original_event" + ], + "@timestamp": "2016-12-07T09:43:23.000Z", + "ecs": { + "version": "1.12.0" + }, + "_tmp": {}, "related": { "ip": [ - "77.179.66.156" + "67.43.156.13" ] }, "http": { @@ -487,37 +521,17 @@ "status_code": 404 } }, - "source": { - "geo": { - "continent_name": "Europe", - "region_iso_code": "DE-RP", - "city_name": "Germersheim", - "country_iso_code": "DE", - "country_name": "Germany", - "region_name": "Rheinland-Pfalz", - "location": { - "lon": 8.3639, - "lat": 49.2231 - } - }, - "as": { - "number": 6805, - "organization": { - "name": "Telefonica Germany" - } - }, - "address": "77.179.66.156", - "ip": "77.179.66.156" - }, "event": { + "ingested": "2021-12-09T13:41:40.653111900Z", + "original": "67.43.156.13 - - [07/Dec/2016:10:43:23 +0100] \"GET /test1 HTTP/1.1\" 404 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", "category": [ "web" ], "type": [ "access" ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event", "outcome": "failure" }, "user_agent": { @@ -532,23 +546,35 @@ "name": "Mac" }, "version": "54.0.2840.98" - }, - "url": { - "original": "/test" } }, { - "@timestamp": "2016-12-07T09:43:23.000Z", "nginx": { "access": { "remote_ip_list": [ - "77.179.66.156" + "127.0.0.1" ] } }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "url": { + "path": "/test1", + "original": "/test1" + }, + "tags": [ + "preserve_original_event" + ], + "@timestamp": "2016-12-07T10:04:37.000Z", + "ecs": { + "version": "1.12.0" + }, + "_tmp": {}, "related": { "ip": [ - "77.179.66.156" + "127.0.0.1" ] }, "http": { @@ -563,37 +589,17 @@ "status_code": 404 } }, - "source": { - "geo": { - "continent_name": "Europe", - "region_iso_code": "DE-RP", - "city_name": "Germersheim", - "country_iso_code": "DE", - "country_name": "Germany", - "region_name": "Rheinland-Pfalz", - "location": { - "lon": 8.3639, - "lat": 49.2231 - } - }, - "as": { - "number": 6805, - "organization": { - "name": "Telefonica Germany" - } - }, - "address": "77.179.66.156", - "ip": "77.179.66.156" - }, "event": { + "ingested": "2021-12-09T13:41:40.653118Z", + "original": "127.0.0.1 - - [07/Dec/2016:11:04:37 +0100] \"GET /test1 HTTP/1.1\" 404 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", "category": [ "web" ], "type": [ "access" ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event", "outcome": "failure" }, "user_agent": { @@ -608,23 +614,35 @@ "name": "Mac" }, "version": "54.0.2840.98" - }, - "url": { - "original": "/test1" } }, { - "@timestamp": "2016-10-25T12:49:33.000Z", "nginx": { "access": { "remote_ip_list": [ - "77.179.66.156" + "127.0.0.1" ] } }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "url": { + "path": "/", + "original": "/" + }, + "tags": [ + "preserve_original_event" + ], + "@timestamp": "2016-12-07T10:04:58.000Z", + "ecs": { + "version": "1.12.0" + }, + "_tmp": {}, "related": { "ip": [ - "77.179.66.156" + "127.0.0.1" ] }, "http": { @@ -634,150 +652,133 @@ "version": "1.1", "response": { "body": { - "bytes": 612 + "bytes": 0 }, - "status_code": 200 + "status_code": 304 } }, - "source": { - "geo": { - "continent_name": "Europe", - "region_iso_code": "DE-RP", - "city_name": "Germersheim", - "country_iso_code": "DE", - "country_name": "Germany", - "region_name": "Rheinland-Pfalz", - "location": { - "lon": 8.3639, - "lat": 49.2231 - } - }, - "as": { - "number": 6805, - "organization": { - "name": "Telefonica Germany" - } - }, - "address": "77.179.66.156", - "ip": "77.179.66.156" - }, "event": { + "ingested": "2021-12-09T13:41:40.653124800Z", + "original": "127.0.0.1 - - [07/Dec/2016:11:04:58 +0100] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", "category": [ "web" ], "type": [ "access" ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event", "outcome": "success" }, "user_agent": { - "name": "Chrome", - "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36", + "name": "Firefox", + "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0", "os": { "name": "Mac OS X", - "version": "10.12.0", - "full": "Mac OS X 10.12.0" + "version": "10.12", + "full": "Mac OS X 10.12" }, "device": { "name": "Mac" }, - "version": "54.0.2840.59" - }, - "url": { - "original": "/" + "version": "49.0." } }, { - "@timestamp": "2016-10-25T12:49:34.000Z", "nginx": { "access": { "remote_ip_list": [ - "77.179.66.156" + "127.0.0.1" ] } }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "url": { + "path": "/", + "original": "/" + }, + "tags": [ + "preserve_original_event" + ], + "@timestamp": "2016-12-07T10:04:59.000Z", + "ecs": { + "version": "1.12.0" + }, + "_tmp": {}, "related": { "ip": [ - "77.179.66.156" + "127.0.0.1" ] }, "http": { "request": { - "method": "get", - "referrer": "http://localhost:8080/" + "method": "get" }, "version": "1.1", "response": { "body": { - "bytes": 571 + "bytes": 0 }, - "status_code": 404 + "status_code": 304 } }, - "source": { - "geo": { - "continent_name": "Europe", - "region_iso_code": "DE-RP", - "city_name": "Germersheim", - "country_iso_code": "DE", - "country_name": "Germany", - "region_name": "Rheinland-Pfalz", - "location": { - "lon": 8.3639, - "lat": 49.2231 - } - }, - "as": { - "number": 6805, - "organization": { - "name": "Telefonica Germany" - } - }, - "address": "77.179.66.156", - "ip": "77.179.66.156" - }, "event": { + "ingested": "2021-12-09T13:41:40.653133100Z", + "original": "127.0.0.1 - - [07/Dec/2016:11:04:59 +0100] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", "category": [ "web" ], "type": [ "access" ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event", - "outcome": "failure" + "outcome": "success" }, "user_agent": { - "name": "Chrome", - "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36", + "name": "Firefox", + "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0", "os": { "name": "Mac OS X", - "version": "10.12.0", - "full": "Mac OS X 10.12.0" + "version": "10.12", + "full": "Mac OS X 10.12" }, "device": { "name": "Mac" }, - "version": "54.0.2840.59" - }, - "url": { - "original": "/favicon.ico" + "version": "49.0." } }, { - "@timestamp": "2016-10-25T12:50:44.000Z", "nginx": { "access": { "remote_ip_list": [ - "77.179.66.156" + "127.0.0.1" ] } }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "url": { + "path": "/taga", + "original": "/taga" + }, + "tags": [ + "preserve_original_event" + ], + "@timestamp": "2016-12-07T10:05:07.000Z", + "ecs": { + "version": "1.12.0" + }, + "_tmp": {}, "related": { "ip": [ - "77.179.66.156" + "127.0.0.1" ] }, "http": { @@ -787,59 +788,36 @@ "version": "1.1", "response": { "body": { - "bytes": 571 + "bytes": 169 }, "status_code": 404 } }, - "source": { - "geo": { - "continent_name": "Europe", - "region_iso_code": "DE-RP", - "city_name": "Germersheim", - "country_iso_code": "DE", - "country_name": "Germany", - "region_name": "Rheinland-Pfalz", - "location": { - "lon": 8.3639, - "lat": 49.2231 - } - }, - "as": { - "number": 6805, - "organization": { - "name": "Telefonica Germany" - } - }, - "address": "77.179.66.156", - "ip": "77.179.66.156" - }, "event": { + "ingested": "2021-12-09T13:41:40.653141500Z", + "original": "127.0.0.1 - - [07/Dec/2016:11:05:07 +0100] \"GET /taga HTTP/1.1\" 404 169 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"\nlessons.example.com 192.168.0.1 - - [09/Jun/2020:12:10:39 -0700] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 206 7648063 \"http://lessons.example.com/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4\" \"Mozilla/5.0 (Linux; Android 5.1.1; KFFOWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/81.2.16 like Chrome/81.0.4044.138 Safari/537.36\"\nlessons.example.com 192.168.0.1 - - [09/Jun/2020:12:15:39 -0700] \"GET /%D0%A0%D1%83%D1%81%D1%81%D0%BA%D0%B0%D1%8F%20%D1%88%D0%BA%D0%BE%D0%BB%D0%B0%20-%20InternetUrok%201%D0%BA%D0%BB%D0%B0%D1%81%D1%81/ HTTP/1.1\" 206 7648063 \"http://lessons.example.com/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4\" \"Mozilla/5.0 (Linux; Android 5.1.1; KFFOWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/81.2.16 like Chrome/81.0.4044.138 Safari/537.36\"", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", "category": [ "web" ], "type": [ "access" ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event", "outcome": "failure" }, "user_agent": { - "name": "Chrome", - "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36", + "name": "Firefox", + "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0", "os": { "name": "Mac OS X", - "version": "10.12.0", - "full": "Mac OS X 10.12.0" + "version": "10.12", + "full": "Mac OS X 10.12" }, "device": { "name": "Mac" }, - "version": "54.0.2840.59" - }, - "url": { - "original": "/adsasd" + "version": "49.0." } } ] diff --git a/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-access-raw.log-config.yml b/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-common-config.yml similarity index 68% rename from test/packages/nginx/data_stream/access/_dev/test/pipeline/test-access-raw.log-config.yml rename to test/packages/nginx/data_stream/access/_dev/test/pipeline/test-common-config.yml index 1de289b7e4..ef369dff36 100644 --- a/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-access-raw.log-config.yml +++ b/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-common-config.yml @@ -1,6 +1,8 @@ +dynamic_fields: + event.ingested: ".*" multiline: first_line_pattern: "^(?:[0-9]{1,3}\\.){3}[0-9]{1,3}" fields: "@timestamp": "2020-04-28T11:07:58.223Z" -dynamic_fields: - url.original: "^/.*$" + tags: + - preserve_original_event diff --git a/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-nginx.log b/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-nginx.log new file mode 100644 index 0000000000..08310b13e2 --- /dev/null +++ b/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-nginx.log @@ -0,0 +1,11 @@ +10.0.0.2, 10.0.0.1, 127.0.0.1 - - [07/Dec/2016:11:05:07 +0100] "GET /ocelot HTTP/1.1" 200 571 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0" +172.17.0.1 - - [29/May/2017:19:02:48 +0000] "GET /stringpatch HTTP/1.1" 404 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-" +10.0.0.2, 10.0.0.1, 67.43.156.14 - - [07/Dec/2016:11:05:07 +0100] "GET /ocelot HTTP/1.1" 200 571 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0" +67.43.156.14 - - [07/Dec/2016:11:05:07 +0100] "GET /ocelot HTTP/1.1" 200 571 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36" +"10.5.102.222, 199.96.1.1, 204.246.1.1" 10.2.1.185 - - [22/Jan/2016:13:18:29 +0000] "GET /assets/xxxx?q=100 HTTP/1.1" 200 25507 "-" "Amazon CloudFront" +2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6, 10.225.192.17 10.2.2.121 - - [30/Dec/2016:06:47:09 +0000] "GET /test.html HTTP/1.1" 404 8571 "-" "Mozilla/5.0 (compatible; Facebot 1.0; https://developers.facebook.com/docs/sharing/webmasters/crawler)" +127.0.0.1 - - [12/Apr/2018:09:48:40 +0200] "" 400 0 "-" "-" +unix: - - [26/Feb/2019:15:39:42 +0100] "hello" 400 173 "-" "-" +localhost - - [29/May/2017:19:02:48 +0000] "GET /test2 HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-" +localhost, localhost - - [29/May/2017:19:02:48 +0000] "GET /test2 HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-" + diff --git a/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-nginx.log-expected.json b/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-nginx.log-expected.json new file mode 100644 index 0000000000..6c162949a2 --- /dev/null +++ b/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-nginx.log-expected.json @@ -0,0 +1,326 @@ +{ + "expected": [ + { + "nginx": { + "access": { + "remote_ip_list": [ + "10.0.0.2", + "10.0.0.1", + "127.0.0.1" + ] + } + }, + "source": { + "address": "10.0.0.2", + "ip": "10.0.0.2" + }, + "url": { + "path": "/ocelot", + "original": "/ocelot" + }, + "tags": [ + "preserve_original_event" + ], + "@timestamp": "2016-12-07T10:05:07.000Z", + "ecs": { + "version": "1.12.0" + }, + "_tmp": {}, + "related": { + "ip": [ + "10.0.0.2" + ] + }, + "http": { + "request": { + "method": "get" + }, + "version": "1.1", + "response": { + "body": { + "bytes": 571 + }, + "status_code": 200 + } + }, + "event": { + "ingested": "2021-12-09T13:41:42.512992900Z", + "original": "10.0.0.2, 10.0.0.1, 127.0.0.1 - - [07/Dec/2016:11:05:07 +0100] \"GET /ocelot HTTP/1.1\" 200 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", + "category": [ + "web" + ], + "type": [ + "access" + ], + "outcome": "success" + }, + "user_agent": { + "name": "Firefox", + "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0", + "os": { + "name": "Mac OS X", + "version": "10.12", + "full": "Mac OS X 10.12" + }, + "device": { + "name": "Mac" + }, + "version": "49.0." + } + }, + { + "nginx": { + "access": { + "remote_ip_list": [ + "172.17.0.1" + ] + } + }, + "source": { + "address": "172.17.0.1", + "ip": "172.17.0.1" + }, + "url": { + "path": "/stringpatch", + "original": "/stringpatch" + }, + "tags": [ + "preserve_original_event" + ], + "@timestamp": "2017-05-29T19:02:48.000Z", + "ecs": { + "version": "1.12.0" + }, + "_tmp": {}, + "related": { + "ip": [ + "172.17.0.1" + ] + }, + "http": { + "request": { + "method": "get" + }, + "version": "1.1", + "response": { + "body": { + "bytes": 612 + }, + "status_code": 404 + } + }, + "event": { + "ingested": "2021-12-09T13:41:42.513002400Z", + "original": "172.17.0.1 - - [29/May/2017:19:02:48 +0000] \"GET /stringpatch HTTP/1.1\" 404 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", + "category": [ + "web" + ], + "type": [ + "access" + ], + "outcome": "failure" + }, + "user_agent": { + "name": "Firefox Alpha", + "original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", + "os": { + "name": "Windows", + "version": "7", + "full": "Windows 7" + }, + "device": { + "name": "Other" + }, + "version": "15.0.a2" + } + }, + { + "nginx": { + "access": { + "remote_ip_list": [ + "10.0.0.2", + "10.0.0.1", + "67.43.156.14" + ] + } + }, + "source": { + "address": "67.43.156.14", + "ip": "67.43.156.14" + }, + "url": { + "path": "/ocelot", + "original": "/ocelot" + }, + "tags": [ + "preserve_original_event" + ], + "@timestamp": "2016-12-07T10:05:07.000Z", + "ecs": { + "version": "1.12.0" + }, + "_tmp": {}, + "related": { + "ip": [ + "67.43.156.14" + ] + }, + "http": { + "request": { + "method": "get" + }, + "version": "1.1", + "response": { + "body": { + "bytes": 571 + }, + "status_code": 200 + } + }, + "event": { + "ingested": "2021-12-09T13:41:42.513008900Z", + "original": "10.0.0.2, 10.0.0.1, 67.43.156.14 - - [07/Dec/2016:11:05:07 +0100] \"GET /ocelot HTTP/1.1\" 200 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", + "category": [ + "web" + ], + "type": [ + "access" + ], + "outcome": "success" + }, + "user_agent": { + "name": "Firefox", + "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0", + "os": { + "name": "Mac OS X", + "version": "10.12", + "full": "Mac OS X 10.12" + }, + "device": { + "name": "Mac" + }, + "version": "49.0." + } + }, + { + "nginx": { + "access": { + "remote_ip_list": [ + "67.43.156.14" + ] + } + }, + "source": { + "address": "67.43.156.14", + "ip": "67.43.156.14" + }, + "url": { + "path": "/ocelot", + "original": "/ocelot" + }, + "tags": [ + "preserve_original_event" + ], + "@timestamp": "2016-12-07T10:05:07.000Z", + "ecs": { + "version": "1.12.0" + }, + "_tmp": {}, + "related": { + "ip": [ + "67.43.156.14" + ] + }, + "http": { + "request": { + "method": "get" + }, + "version": "1.1", + "response": { + "body": { + "bytes": 571 + }, + "status_code": 200 + } + }, + "event": { + "ingested": "2021-12-09T13:41:42.513013400Z", + "original": "67.43.156.14 - - [07/Dec/2016:11:05:07 +0100] \"GET /ocelot HTTP/1.1\" 200 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36\"\n\"10.5.102.222, 199.96.1.1, 204.246.1.1\" 10.2.1.185 - - [22/Jan/2016:13:18:29 +0000] \"GET /assets/xxxx?q=100 HTTP/1.1\" 200 25507 \"-\" \"Amazon CloudFront\"\n2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6, 10.225.192.17 10.2.2.121 - - [30/Dec/2016:06:47:09 +0000] \"GET /test.html HTTP/1.1\" 404 8571 \"-\" \"Mozilla/5.0 (compatible; Facebot 1.0; https://developers.facebook.com/docs/sharing/webmasters/crawler)\"", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", + "category": [ + "web" + ], + "type": [ + "access" + ], + "outcome": "success" + }, + "user_agent": { + "name": "Chrome", + "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36", + "os": { + "name": "Mac OS X", + "version": "10.14.0", + "full": "Mac OS X 10.14.0" + }, + "device": { + "name": "Mac" + }, + "version": "70.0.3538.102" + } + }, + { + "@timestamp": "2018-04-12T07:48:40.000Z", + "ecs": { + "version": "1.12.0" + }, + "nginx": { + "access": { + "remote_ip_list": [ + "127.0.0.1" + ] + } + }, + "related": { + "ip": [ + "127.0.0.1" + ] + }, + "http": { + "response": { + "body": { + "bytes": 0 + }, + "status_code": 400 + } + }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2021-12-09T13:41:42.513018600Z", + "original": "127.0.0.1 - - [12/Apr/2018:09:48:40 +0200] \"\" 400 0 \"-\" \"-\"\nunix: - - [26/Feb/2019:15:39:42 +0100] \"hello\" 400 173 \"-\" \"-\"\nlocalhost - - [29/May/2017:19:02:48 +0000] \"GET /test2 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"\nlocalhost, localhost - - [29/May/2017:19:02:48 +0000] \"GET /test2 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"\n", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", + "category": [ + "web" + ], + "type": [ + "access" + ], + "outcome": "failure" + }, + "tags": [ + "preserve_original_event" + ] + } + ] +} \ No newline at end of file diff --git a/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-test-with-host.log b/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-test-with-host.log new file mode 100644 index 0000000000..be64f1d3bc --- /dev/null +++ b/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-test-with-host.log @@ -0,0 +1,10 @@ +example.com 10.0.0.2, 10.0.0.1, 127.0.0.1 - - [07/Dec/2016:11:05:07 +0100] "GET /ocelot HTTP/1.1" 200 571 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0" +example.com 172.17.0.1 - - [29/May/2017:19:02:48 +0000] "GET /stringpatch HTTP/1.1" 404 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-" +example.com 10.0.0.2, 10.0.0.1, 67.43.156.14 - - [07/Dec/2016:11:05:07 +0100] "GET /ocelot HTTP/1.1" 200 571 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0" +example.com:80 67.43.156.14 - - [07/Dec/2016:11:05:07 +0100] "GET /ocelot HTTP/1.1" 200 571 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36" +example.com:80 "10.5.102.222, 199.96.1.1, 204.246.1.1" 10.2.1.185 - - [22/Jan/2016:13:18:29 +0000] "GET /assets/xxxx?q=100 HTTP/1.1" 200 25507 "-" "Amazon CloudFront" +67.43.156.15 2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6, 10.225.192.17 10.2.2.121 - - [30/Dec/2016:06:47:09 +0000] "GET /test.html HTTP/1.1" 404 8571 "-" "Mozilla/5.0 (compatible; Facebot 1.0; https://developers.facebook.com/docs/sharing/webmasters/crawler)" +67.43.156.15:80 127.0.0.1 - - [12/Apr/2018:09:48:40 +0200] "" 400 0 "-" "-" +example.com:80 unix: - - [26/Feb/2019:15:39:42 +0100] "hello" 400 173 "-" "-" +67.43.156.15 localhost - - [29/May/2017:19:02:48 +0000] "GET /test2 HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-" +example.com localhost, localhost - - [29/May/2017:19:02:48 +0000] "GET /test2 HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-" diff --git a/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-test-with-host.log-expected.json b/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-test-with-host.log-expected.json new file mode 100644 index 0000000000..208427cae3 --- /dev/null +++ b/test/packages/nginx/data_stream/access/_dev/test/pipeline/test-test-with-host.log-expected.json @@ -0,0 +1,284 @@ +{ + "expected": [ + { + "nginx": { + "access": { + "remote_ip_list": [ + "10.0.0.2", + "10.0.0.1", + "127.0.0.1" + ] + } + }, + "destination": { + "domain": "example.com" + }, + "source": { + "address": "10.0.0.2", + "ip": "10.0.0.2" + }, + "url": { + "path": "/ocelot", + "original": "/ocelot", + "domain": "example.com" + }, + "tags": [ + "preserve_original_event" + ], + "@timestamp": "2016-12-07T10:05:07.000Z", + "ecs": { + "version": "1.12.0" + }, + "_tmp": {}, + "related": { + "ip": [ + "10.0.0.2" + ] + }, + "http": { + "request": { + "method": "get" + }, + "version": "1.1", + "response": { + "body": { + "bytes": 571 + }, + "status_code": 200 + } + }, + "event": { + "ingested": "2021-12-09T13:41:43.172715Z", + "original": "example.com 10.0.0.2, 10.0.0.1, 127.0.0.1 - - [07/Dec/2016:11:05:07 +0100] \"GET /ocelot HTTP/1.1\" 200 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"\nexample.com 172.17.0.1 - - [29/May/2017:19:02:48 +0000] \"GET /stringpatch HTTP/1.1\" 404 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"\nexample.com 10.0.0.2, 10.0.0.1, 67.43.156.14 - - [07/Dec/2016:11:05:07 +0100] \"GET /ocelot HTTP/1.1\" 200 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"\nexample.com:80 67.43.156.14 - - [07/Dec/2016:11:05:07 +0100] \"GET /ocelot HTTP/1.1\" 200 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36\"\nexample.com:80 \"10.5.102.222, 199.96.1.1, 204.246.1.1\" 10.2.1.185 - - [22/Jan/2016:13:18:29 +0000] \"GET /assets/xxxx?q=100 HTTP/1.1\" 200 25507 \"-\" \"Amazon CloudFront\"", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", + "category": [ + "web" + ], + "type": [ + "access" + ], + "outcome": "success" + }, + "user_agent": { + "name": "Firefox", + "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0", + "os": { + "name": "Mac OS X", + "version": "10.12", + "full": "Mac OS X 10.12" + }, + "device": { + "name": "Mac" + }, + "version": "49.0." + } + }, + { + "nginx": { + "access": { + "remote_ip_list": [ + "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6", + "10.225.192.17", + "10.2.2.121" + ] + } + }, + "destination": { + "ip": "67.43.156.15" + }, + "source": { + "geo": { + "continent_name": "Europe", + "country_name": "Denmark", + "location": { + "lon": 10.0, + "lat": 56.0 + }, + "country_iso_code": "DK" + }, + "as": { + "number": 62121, + "organization": { + "name": "Christian Ebsen ApS" + } + }, + "address": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6", + "ip": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6" + }, + "url": { + "path": "/test.html", + "extension": "html", + "original": "/test.html" + }, + "tags": [ + "preserve_original_event" + ], + "@timestamp": "2016-12-30T06:47:09.000Z", + "ecs": { + "version": "1.12.0" + }, + "_tmp": {}, + "related": { + "ip": [ + "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6", + "67.43.156.15" + ] + }, + "http": { + "request": { + "method": "get" + }, + "version": "1.1", + "response": { + "body": { + "bytes": 8571 + }, + "status_code": 404 + } + }, + "event": { + "ingested": "2021-12-09T13:41:43.172728700Z", + "original": "67.43.156.15 2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6, 10.225.192.17 10.2.2.121 - - [30/Dec/2016:06:47:09 +0000] \"GET /test.html HTTP/1.1\" 404 8571 \"-\" \"Mozilla/5.0 (compatible; Facebot 1.0; https://developers.facebook.com/docs/sharing/webmasters/crawler)\"", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", + "category": [ + "web" + ], + "type": [ + "access" + ], + "outcome": "failure" + }, + "user_agent": { + "name": "Facebot", + "original": "Mozilla/5.0 (compatible; Facebot 1.0; https://developers.facebook.com/docs/sharing/webmasters/crawler)", + "device": { + "name": "Spider" + }, + "version": "1.0" + } + }, + { + "@timestamp": "2018-04-12T07:48:40.000Z", + "ecs": { + "version": "1.12.0" + }, + "nginx": { + "access": { + "remote_ip_list": [ + "127.0.0.1" + ] + } + }, + "related": { + "ip": [ + "127.0.0.1", + "67.43.156.15" + ] + }, + "destination": { + "port": 80, + "ip": "67.43.156.15" + }, + "http": { + "response": { + "body": { + "bytes": 0 + }, + "status_code": 400 + } + }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2021-12-09T13:41:43.172732700Z", + "original": "67.43.156.15:80 127.0.0.1 - - [12/Apr/2018:09:48:40 +0200] \"\" 400 0 \"-\" \"-\"\nexample.com:80 unix: - - [26/Feb/2019:15:39:42 +0100] \"hello\" 400 173 \"-\" \"-\"", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", + "category": [ + "web" + ], + "type": [ + "access" + ], + "outcome": "failure" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "nginx": { + "access": { + "remote_ip_list": [ + "localhost" + ] + } + }, + "destination": { + "ip": "67.43.156.15" + }, + "source": { + "address": "localhost" + }, + "url": { + "path": "/test2", + "original": "/test2" + }, + "tags": [ + "preserve_original_event" + ], + "@timestamp": "2017-05-29T19:02:48.000Z", + "ecs": { + "version": "1.12.0" + }, + "_tmp": {}, + "related": { + "ip": [ + "67.43.156.15" + ] + }, + "http": { + "request": { + "method": "get" + }, + "version": "1.1", + "response": { + "body": { + "bytes": 612 + }, + "status_code": 200 + } + }, + "event": { + "ingested": "2021-12-09T13:41:43.172738700Z", + "original": "67.43.156.15 localhost - - [29/May/2017:19:02:48 +0000] \"GET /test2 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"\nexample.com localhost, localhost - - [29/May/2017:19:02:48 +0000] \"GET /test2 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", + "category": [ + "web" + ], + "type": [ + "access" + ], + "outcome": "success" + }, + "user_agent": { + "name": "Firefox Alpha", + "original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", + "os": { + "name": "Windows", + "version": "7", + "full": "Windows 7" + }, + "device": { + "name": "Other" + }, + "version": "15.0.a2" + } + } + ] +} \ No newline at end of file diff --git a/test/packages/nginx/data_stream/access/_dev/test/system/test-default-config.yml b/test/packages/nginx/data_stream/access/_dev/test/system/test-default-config.yml new file mode 100644 index 0000000000..0f73c5dd9e --- /dev/null +++ b/test/packages/nginx/data_stream/access/_dev/test/system/test-default-config.yml @@ -0,0 +1,5 @@ +vars: ~ +data_stream: + vars: + paths: + - "{{SERVICE_LOGS_DIR}}/access.log*" diff --git a/test/packages/nginx/data_stream/access/agent/stream/httpjson.yml.hbs b/test/packages/nginx/data_stream/access/agent/stream/httpjson.yml.hbs new file mode 100644 index 0000000000..27d1775b51 --- /dev/null +++ b/test/packages/nginx/data_stream/access/agent/stream/httpjson.yml.hbs @@ -0,0 +1,63 @@ +config_version: 2 +interval: {{interval}} +{{#unless token}} +{{#if username}} +{{#if password}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +{{/if}} +{{/if}} +{{/unless}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +{{#unless username}} +{{#unless password}} +{{#if token}} + - set: + target: header.Authorization + value: {{token}} +{{/if}} +{{/unless}} +{{/unless}} +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} \ No newline at end of file diff --git a/test/packages/nginx/data_stream/access/agent/stream/stream.yml.hbs b/test/packages/nginx/data_stream/access/agent/stream/stream.yml.hbs index d0d5ed877c..531705c5f3 100644 --- a/test/packages/nginx/data_stream/access/agent/stream/stream.yml.hbs +++ b/test/packages/nginx/data_stream/access/agent/stream/stream.yml.hbs @@ -2,6 +2,19 @@ paths: {{#each paths}} - {{this}} {{/each}} +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} exclude_files: [".gz$"] processors: +{{#if processors}} +{{processors}} +{{/if}} - add_locale: ~ diff --git a/test/packages/nginx/data_stream/access/elasticsearch/ingest_pipeline/default.yml b/test/packages/nginx/data_stream/access/elasticsearch/ingest_pipeline/default.yml index 9de5d5e7c4..054d992be6 100644 --- a/test/packages/nginx/data_stream/access/elasticsearch/ingest_pipeline/default.yml +++ b/test/packages/nginx/data_stream/access/elasticsearch/ingest_pipeline/default.yml @@ -2,167 +2,225 @@ description: Pipeline for parsing Nginx access logs. Requires the geoip and user_agent plugins. processors: -- grok: - field: message - patterns: - - (%{NGINX_HOST} )?"?(?:%{NGINX_ADDRESS_LIST:nginx.access.remote_ip_list}|%{NOTSPACE:source.address}) - - (-|%{DATA:user.name}) \[%{HTTPDATE:nginx.access.time}\] "%{DATA:nginx.access.info}" - %{NUMBER:http.response.status_code:long} %{NUMBER:http.response.body.bytes:long} - "(-|%{DATA:http.request.referrer})" "(-|%{DATA:user_agent.original})" - pattern_definitions: - NGINX_HOST: (?:%{IP:destination.ip}|%{NGINX_NOTSEPARATOR:destination.domain})(:%{NUMBER:destination.port})? - NGINX_NOTSEPARATOR: "[^\t ,:]+" - NGINX_ADDRESS_LIST: (?:%{IP}|%{WORD})("?,?\s*(?:%{IP}|%{WORD}))* - ignore_missing: true -- grok: - field: nginx.access.info - patterns: - - '%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}' - - "" - ignore_missing: true -- remove: - field: nginx.access.info -- split: - field: nginx.access.remote_ip_list - separator: '"?,?\s+' - ignore_missing: true -- split: - field: nginx.access.origin - separator: '"?,?\s+' - ignore_missing: true -- set: - field: source.address - if: ctx.source?.address == null - value: "" -- script: - if: ctx.nginx?.access?.remote_ip_list != null && ctx.nginx.access.remote_ip_list.length > 0 - lang: painless - source: >- - boolean isPrivate(def dot, def ip) { - try { - StringTokenizer tok = new StringTokenizer(ip, dot); - int firstByte = Integer.parseInt(tok.nextToken()); - int secondByte = Integer.parseInt(tok.nextToken()); - if (firstByte == 10) { - return true; + - pipeline: + if: ctx.message.startsWith('{') + name: '{{ IngestPipeline "third-party" }}' + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + - set: + field: ecs.version + value: '1.12.0' + - rename: + field: message + target_field: event.original + ignore_missing: true + - grok: + field: event.original + patterns: + - (%{NGINX_HOST} )?"?(?:%{NGINX_ADDRESS_LIST:nginx.access.remote_ip_list}|%{NOTSPACE:source.address}) + - (-|%{DATA:user.name}) \[%{HTTPDATE:nginx.access.time}\] "%{DATA:nginx.access.info}" + %{NUMBER:http.response.status_code:long} %{NUMBER:http.response.body.bytes:long} + "(-|%{DATA:http.request.referrer})" "(-|%{DATA:user_agent.original})" + pattern_definitions: + NGINX_HOST: (?:%{IP:destination.ip}|%{NGINX_NOTSEPARATOR:destination.domain})(:%{NUMBER:destination.port})? + NGINX_NOTSEPARATOR: "[^\t ,:]+" + NGINX_ADDRESS_LIST: (?:%{IP}|%{WORD})("?,?\s*(?:%{IP}|%{WORD}))* + ignore_missing: true + - grok: + field: nginx.access.info + patterns: + - '%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}' + - "" + ignore_missing: true + - uri_parts: + field: _tmp.url_orig + ignore_failure: true + - set: + field: url.domain + value: "{{destination.domain}}" + if: ctx.url?.domain == null && ctx.destination?.domain != null + - remove: + field: + - nginx.access.info + - _tmp.url_orig + ignore_missing: true + - split: + field: nginx.access.remote_ip_list + separator: '"?,?\s+' + ignore_missing: true + - split: + field: nginx.access.origin + separator: '"?,?\s+' + ignore_missing: true + - set: + field: source.address + if: ctx.source?.address == null + value: "" + - script: + if: ctx.nginx?.access?.remote_ip_list != null && ctx.nginx.access.remote_ip_list.length > 0 + lang: painless + source: >- + boolean isPrivate(def dot, def ip) { + try { + StringTokenizer tok = new StringTokenizer(ip, dot); + int firstByte = Integer.parseInt(tok.nextToken()); + int secondByte = Integer.parseInt(tok.nextToken()); + if (firstByte == 10) { + return true; + } + if (firstByte == 192 && secondByte == 168) { + return true; + } + if (firstByte == 172 && secondByte >= 16 && secondByte <= 31) { + return true; + } + if (firstByte == 127) { + return true; + } + return false; + } + catch (Exception e) { + return false; } - if (firstByte == 192 && secondByte == 168) { - return true; + } + try { + ctx.source.address = null; + if (ctx.nginx.access.remote_ip_list == null) { + return; } - if (firstByte == 172 && secondByte >= 16 && secondByte <= 31) { - return true; + def found = false; + for (def item : ctx.nginx.access.remote_ip_list) { + if (!isPrivate(params.dot, item)) { + ctx.source.address = item; + found = true; + break; + } } - if (firstByte == 127) { - return true; + if (!found) { + ctx.source.address = ctx.nginx.access.remote_ip_list[0]; } - return false; } catch (Exception e) { - return false; - } - } - try { - ctx.source.address = null; - if (ctx.nginx.access.remote_ip_list == null) { - return; + ctx.source.address = null; } - def found = false; - for (def item : ctx.nginx.access.remote_ip_list) { - if (!isPrivate(params.dot, item)) { - ctx.source.address = item; - found = true; - break; + params: + dot: . + - remove: + field: source.address + if: ctx.source.address == null + - grok: + field: source.address + patterns: + - ^%{IP:source.ip}$ + ignore_failure: true + - remove: + field: event.created + ignore_missing: true + ignore_failure: true + - rename: + field: '@timestamp' + target_field: event.created + - convert: + field: destination.port + type: long + if: ctx.destination?.port != null + - date: + field: nginx.access.time + target_field: '@timestamp' + formats: + - dd/MMM/yyyy:H:m:s Z + on_failure: + - append: + field: error.message + value: '{{ _ingest.on_failure_message }}' + - remove: + field: nginx.access.time + - user_agent: + field: user_agent.original + ignore_missing: true + - geoip: + field: source.ip + target_field: source.geo + ignore_missing: true + - geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true + - rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true + - rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true + - set: + field: event.kind + value: event + - append: + field: event.category + value: web + - append: + field: event.type + value: access + - set: + field: event.outcome + value: success + if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400" + - set: + field: event.outcome + value: failure + if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400" + - lowercase: + field: http.request.method + ignore_missing: true + - append: + field: related.ip + value: "{{source.ip}}" + if: "ctx?.source?.ip != null" + - append: + field: related.ip + value: "{{destination.ip}}" + if: "ctx?.destination?.ip != null" + - append: + field: related.user + value: "{{user.name}}" + if: "ctx?.user?.name != null" + - script: + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } } + map.values().removeIf(v -> v == null); } - if (!found) { - ctx.source.address = ctx.nginx.access.remote_ip_list[0]; + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } } - } - catch (Exception e) { - ctx.source.address = null; - } - params: - dot: . -- remove: - field: source.address - if: ctx.source.address == null -- grok: - field: source.address - patterns: - - ^%{IP:source.ip}$ - ignore_failure: true -- remove: - field: message -- rename: - field: '@timestamp' - target_field: event.created -- date: - field: nginx.access.time - target_field: '@timestamp' - formats: - - dd/MMM/yyyy:H:m:s Z - on_failure: - - append: - field: error.message - value: '{{ _ingest.on_failure_message }}' -- remove: - field: nginx.access.time -- user_agent: - field: user_agent.original - ignore_missing: true -- geoip: - field: source.ip - target_field: source.geo - ignore_missing: true -- geoip: - database_file: GeoLite2-ASN.mmdb - field: source.ip - target_field: source.as - properties: - - asn - - organization_name - ignore_missing: true -- rename: - field: source.as.asn - target_field: source.as.number - ignore_missing: true -- rename: - field: source.as.organization_name - target_field: source.as.organization.name - ignore_missing: true -- set: - field: event.kind - value: event -- append: - field: event.category - value: web -- append: - field: event.type - value: access -- set: - field: event.outcome - value: success - if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400" -- set: - field: event.outcome - value: failure - if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400" -- lowercase: - field: http.request.method - ignore_missing: true -- append: - field: related.ip - value: "{{source.ip}}" - if: "ctx?.source?.ip != null" -- append: - field: related.ip - value: "{{destination.ip}}" - if: "ctx?.destination?.ip != null" -- append: - field: related.user - value: "{{user.name}}" - if: "ctx?.user?.name != null" + handleMap(ctx); + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: -- set: - field: error.message - value: '{{ _ingest.on_failure_message }}' + - set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/test/packages/nginx/data_stream/access/elasticsearch/ingest_pipeline/third-party.yml b/test/packages/nginx/data_stream/access/elasticsearch/ingest_pipeline/third-party.yml new file mode 100644 index 0000000000..da7fce1118 --- /dev/null +++ b/test/packages/nginx/data_stream/access/elasticsearch/ingest_pipeline/third-party.yml @@ -0,0 +1,42 @@ +--- +description: Pipeline for parsing nginx logs from third party api +processors: + - json: + field: message + target_field: json + - drop: + if: ctx.json?.result == null + - fingerprint: + fields: + - json.result._cd + - json.result._indextime + - json.result._raw + - json.result._time + - json.result.host + - json.result.source + target_field: '_id' + ignore_missing: true + - set: + copy_from: json.result._raw + field: message + ignore_empty_value: true + - set: + copy_from: json.result.host + field: host.name + ignore_empty_value: true + - set: + copy_from: json.result.source + field: file.path + ignore_empty_value: true + - remove: + field: + - json + ignore_missing: true +on_failure: + - append: + field: error.message + value: >- + error in third-party pipeline: + error in [{{_ingest.on_failure_processor_type}}] processor{{#_ingest.on_failure_processor_tag}} + with tag [{{_ingest.on_failure_processor_tag }}]{{/_ingest.on_failure_processor_tag}} + {{ _ingest.on_failure_message }} diff --git a/test/packages/nginx/data_stream/access/fields/agent.yml b/test/packages/nginx/data_stream/access/fields/agent.yml new file mode 100644 index 0000000000..e313ec8287 --- /dev/null +++ b/test/packages/nginx/data_stream/access/fields/agent.yml @@ -0,0 +1,204 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + +- name: input.type + type: keyword + description: Input type +- name: log.offset + type: long + description: Log offset diff --git a/test/packages/nginx/data_stream/access/fields/base-fields.yml b/test/packages/nginx/data_stream/access/fields/base-fields.yml index 502030b8c3..f7f1000887 100644 --- a/test/packages/nginx/data_stream/access/fields/base-fields.yml +++ b/test/packages/nginx/data_stream/access/fields/base-fields.yml @@ -1,13 +1,20 @@ - name: data_stream.type - external: ecs + type: constant_keyword description: Data stream type. - value: logs - name: data_stream.dataset - external: ecs + type: constant_keyword description: Data stream dataset. - value: nginx.access - name: data_stream.namespace - external: ecs + type: constant_keyword + description: Data stream namespace. - name: '@timestamp' type: date description: Event timestamp. +- name: event.module + type: constant_keyword + description: Event module + value: nginx +- name: event.dataset + type: constant_keyword + description: Event dataset + value: nginx.access diff --git a/test/packages/nginx/data_stream/access/fields/ecs.yml b/test/packages/nginx/data_stream/access/fields/ecs.yml index 89b34401dd..1888586bd9 100644 --- a/test/packages/nginx/data_stream/access/fields/ecs.yml +++ b/test/packages/nginx/data_stream/access/fields/ecs.yml @@ -1,103 +1,78 @@ -- name: http - title: HTTP - type: group - fields: - - name: request.method - external: ecs - - name: request.referrer - external: ecs - - name: response.body.bytes - external: ecs - - name: response.status_code - external: ecs - - name: version - external: ecs -- name: source - title: Source - type: group - fields: - - name: address - external: ecs - - name: geo.city_name - external: ecs - - name: geo.continent_name - external: ecs - - name: geo.country_iso_code - external: ecs - - name: geo.region_iso_code - external: ecs - - name: geo.region_name - external: ecs - - name: ip - level: core - type: ip - description: IP address of the source (IPv4 or IPv6). -- name: url - type: group - fields: - - name: original - external: ecs -- name: user - type: group - fields: - - name: name - external: ecs -- name: user_agent - title: User agent - type: group - fields: - - name: device.name - level: extended - type: keyword - description: Name of the device. - ignore_above: 1024 - - name: name - level: extended - type: keyword - description: Name of the user agent. - ignore_above: 1024 - - name: original - level: extended - type: keyword - description: Unparsed user_agent string. - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - - name: os.name - level: extended - type: keyword - description: Operating system name, without the version. - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false -- name: related - description: 'This field set is meant to facilitate pivoting around a piece of data.' - type: group - fields: - - name: ip - type: ip - description: All of the IPs seen on your event. -- name: event.category - external: ecs -- name: event.created - external: ecs -- name: user_agent.os.full - external: ecs -- name: user_agent.os.version - type: keyword - description: Operating system version as a raw string. -- name: user_agent.version - external: ecs -- name: source.geo.location +- external: ecs + name: destination.domain +- external: ecs + name: destination.ip +- external: ecs + name: destination.port +- external: ecs + name: ecs.version +- external: ecs + name: host.ip +- external: ecs + name: http.request.method +- external: ecs + name: http.request.referrer +- external: ecs + name: http.response.body.bytes +- external: ecs + name: http.response.status_code +- external: ecs + name: http.version +- external: ecs + name: log.file.path +- external: ecs + name: related.ip +- external: ecs + name: source.address +- external: ecs + name: source.as.number +- external: ecs + name: source.as.organization.name +- external: ecs + name: source.geo.city_name +- external: ecs + name: source.geo.continent_name +- external: ecs + name: source.geo.country_iso_code +- external: ecs + name: source.geo.country_name +- description: Longitude and latitude. + level: core + name: source.geo.location type: geo_point -- name: source.geo.country_name - external: ecs -- name: source.as.number - external: ecs -- name: source.as.organization.name - external: ecs +- external: ecs + name: source.geo.region_iso_code +- external: ecs + name: source.geo.region_name +- external: ecs + name: source.ip +- external: ecs + name: tags +- external: ecs + name: url.domain +- external: ecs + name: url.extension +- external: ecs + name: url.fragment +- external: ecs + name: url.original +- external: ecs + name: url.path +- external: ecs + name: url.scheme +- external: ecs + name: user.name +- external: ecs + name: user_agent.device.name +- external: ecs + name: user_agent.name +- external: ecs + name: user_agent.original +- external: ecs + name: user_agent.os.full +- external: ecs + name: user_agent.os.name +- external: ecs + name: user_agent.os.version +- external: ecs + name: user_agent.version diff --git a/test/packages/nginx/data_stream/access/fields/fields.yml b/test/packages/nginx/data_stream/access/fields/fields.yml index f64662f0ff..2a7290273e 100644 --- a/test/packages/nginx/data_stream/access/fields/fields.yml +++ b/test/packages/nginx/data_stream/access/fields/fields.yml @@ -2,10 +2,11 @@ type: group fields: - name: remote_ip_list - type: keyword + type: array description: | An array of remote IP addresses. It is a list because it is common to include, besides the client IP address, IP addresses from headers like `X-Forwarded-For`. Real source IP is restored to `source.ip`. - - name: user_agent - type: group - name: geoip type: group +- name: event.created + type: date + description: Date/time when the event was first read by an agent, or by your pipeline. diff --git a/test/packages/nginx/data_stream/access/manifest.yml b/test/packages/nginx/data_stream/access/manifest.yml index 410f19978a..4892e1f828 100644 --- a/test/packages/nginx/data_stream/access/manifest.yml +++ b/test/packages/nginx/data_stream/access/manifest.yml @@ -1,6 +1,5 @@ title: Nginx access logs type: logs -release: experimental streams: - input: logfile vars: @@ -12,5 +11,73 @@ streams: show_user: true default: - /var/log/nginx/access.log* + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - nginx-access + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + title: Nginx access logs description: Collect Nginx access logs + - input: httpjson + title: Nginx access logs via Splunk Enterprise REST API + description: Collect Nginx access logs via Splunk Enterprise REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query Splunk Enterprise REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Splunk search string + show_user: true + required: true + default: search sourcetype=nginx:plus:access + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded + - nginx-access + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: >- + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. diff --git a/test/packages/nginx/data_stream/access/sample_event.json b/test/packages/nginx/data_stream/access/sample_event.json new file mode 100644 index 0000000000..b44215034b --- /dev/null +++ b/test/packages/nginx/data_stream/access/sample_event.json @@ -0,0 +1,106 @@ +{ + "agent": { + "hostname": "a73e7856c209", + "name": "a73e7856c209", + "id": "3987d2b3-b40a-4aa0-99fc-478f9d7079ea", + "ephemeral_id": "6d41da1c-5f71-4bd4-b326-a8913bfaa884", + "type": "filebeat", + "version": "7.11.0" + }, + "nginx": { + "access": { + "remote_ip_list": [ + "127.0.0.1" + ] + } + }, + "log": { + "file": { + "path": "/tmp/service_logs/access.log" + }, + "offset": 0 + }, + "elastic_agent": { + "id": "5ca3af72-37c3-48b6-92e8-176d154bb66f", + "version": "7.11.0", + "snapshot": true + }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "url": { + "original": "/server-status" + }, + "input": { + "type": "log" + }, + "@timestamp": "2020-12-03T11:41:57.000Z", + "ecs": { + "version": "1.6.0" + }, + "related": { + "ip": [ + "127.0.0.1" + ] + }, + "data_stream": { + "namespace": "ep", + "type": "logs", + "dataset": "nginx.access" + }, + "host": { + "hostname": "a73e7856c209", + "os": { + "kernel": "4.9.184-linuxkit", + "codename": "Core", + "name": "CentOS Linux", + "family": "redhat", + "version": "7 (Core)", + "platform": "centos" + }, + "containerized": true, + "ip": [ + "192.168.80.6" + ], + "name": "a73e7856c209", + "id": "06c26569966fd125c15acac5d7feffb6", + "mac": [ + "02:42:c0:a8:50:06" + ], + "architecture": "x86_64" + }, + "http": { + "request": { + "method": "get" + }, + "response": { + "status_code": 200, + "body": { + "bytes": 97 + } + }, + "version": "1.1" + }, + "event": { + "timezone": "+00:00", + "created": "2020-12-03T11:42:17.116Z", + "kind": "event", + "category": [ + "web" + ], + "type": [ + "access" + ], + "dataset": "nginx.access", + "outcome": "success" + }, + "user_agent": { + "original": "curl/7.64.0", + "name": "curl", + "device": { + "name": "Other" + }, + "version": "7.64.0" + } +} \ No newline at end of file diff --git a/test/packages/nginx/data_stream/error/_dev/test/pipeline/test-common-config.yml b/test/packages/nginx/data_stream/error/_dev/test/pipeline/test-common-config.yml deleted file mode 100644 index 28530b51be..0000000000 --- a/test/packages/nginx/data_stream/error/_dev/test/pipeline/test-common-config.yml +++ /dev/null @@ -1,2 +0,0 @@ -multiline: - first_line_pattern: "^\\d{4}\\/\\d{2}\\/\\d{2} " diff --git a/test/packages/nginx/data_stream/error/_dev/test/pipeline/test-error-raw.log-config.yml b/test/packages/nginx/data_stream/error/_dev/test/pipeline/test-error-raw.log-config.yml index 9ca2cf7089..298697a0c0 100644 --- a/test/packages/nginx/data_stream/error/_dev/test/pipeline/test-error-raw.log-config.yml +++ b/test/packages/nginx/data_stream/error/_dev/test/pipeline/test-error-raw.log-config.yml @@ -1,2 +1,8 @@ +multiline: + first_line_pattern: "^\\d{4}\\/\\d{2}\\/\\d{2} " fields: "@timestamp": "2020-04-28T11:07:58.223Z" + tags: + - preserve_original_event +dynamic_fields: + event.ingested: ".*" diff --git a/test/packages/nginx/data_stream/error/_dev/test/pipeline/test-error-raw.log-expected.json b/test/packages/nginx/data_stream/error/_dev/test/pipeline/test-error-raw.log-expected.json index 96e34e0c60..76feff53fe 100644 --- a/test/packages/nginx/data_stream/error/_dev/test/pipeline/test-error-raw.log-expected.json +++ b/test/packages/nginx/data_stream/error/_dev/test/pipeline/test-error-raw.log-expected.json @@ -8,6 +8,9 @@ } }, "@timestamp": "2016-10-25T14:49:34.000Z", + "ecs": { + "version": "1.12.0" + }, "nginx": { "error": { "connection_id": 1 @@ -16,8 +19,9 @@ "log": { "level": "error" }, - "message": "open() \"/usr/local/Cellar/nginx/1.10.2_1/html/favicon.ico\" failed (2: No such file or directory), client: 127.0.0.1, server: localhost, request: \"GET /favicon.ico HTTP/1.1\", host: \"localhost:8080\", referrer: \"http://localhost:8080/\"", "event": { + "ingested": "2021-12-09T13:41:43.902411600Z", + "original": "2016/10/25 14:49:34 [error] 54053#0: *1 open() \"/usr/local/Cellar/nginx/1.10.2_1/html/favicon.ico\" failed (2: No such file or directory), client: 127.0.0.1, server: localhost, request: \"GET /favicon.ico HTTP/1.1\", host: \"localhost:8080\", referrer: \"http://localhost:8080/\"", "category": [ "web" ], @@ -26,7 +30,11 @@ ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" - } + }, + "message": "open() \"/usr/local/Cellar/nginx/1.10.2_1/html/favicon.ico\" failed (2: No such file or directory), client: 127.0.0.1, server: localhost, request: \"GET /favicon.ico HTTP/1.1\", host: \"localhost:8080\", referrer: \"http://localhost:8080/\"", + "tags": [ + "preserve_original_event" + ] }, { "process": { @@ -36,6 +44,9 @@ } }, "@timestamp": "2016-10-25T14:50:44.000Z", + "ecs": { + "version": "1.12.0" + }, "nginx": { "error": { "connection_id": 3 @@ -44,8 +55,9 @@ "log": { "level": "error" }, - "message": "open() \"/usr/local/Cellar/nginx/1.10.2_1/html/adsasd\" failed (2: No such file or directory), client: 127.0.0.1, server: localhost, request: \"GET /adsasd HTTP/1.1\", host: \"localhost:8080\"", "event": { + "ingested": "2021-12-09T13:41:43.902415500Z", + "original": "2016/10/25 14:50:44 [error] 54053#0: *3 open() \"/usr/local/Cellar/nginx/1.10.2_1/html/adsasd\" failed (2: No such file or directory), client: 127.0.0.1, server: localhost, request: \"GET /adsasd HTTP/1.1\", host: \"localhost:8080\"", "category": [ "web" ], @@ -54,7 +66,11 @@ ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" - } + }, + "message": "open() \"/usr/local/Cellar/nginx/1.10.2_1/html/adsasd\" failed (2: No such file or directory), client: 127.0.0.1, server: localhost, request: \"GET /adsasd HTTP/1.1\", host: \"localhost:8080\"", + "tags": [ + "preserve_original_event" + ] }, { "process": { @@ -64,6 +80,9 @@ } }, "@timestamp": "2019-10-30T23:26:34.000Z", + "ecs": { + "version": "1.12.0" + }, "nginx": { "error": { "connection_id": 180289 @@ -72,8 +91,9 @@ "log": { "level": "error" }, - "message": "FastCGI sent in stderr: \"PHP message: PHP Warning: Declaration of FEE_Field_Terms::wrap($content, $taxonomy, $before, $sep, $after) should be compatible with FEE_Field_Post::wrap($content, $post_id = 0) in /var/www/xxx/web/wp-content/plugins/front-end-editor/php/fields/post.php on line 0\nPHP message: PHP Warning: Declaration of FEE_Field_Tags::wrap($content, $before, $sep, $after) should be compatible with FEE_Field_Terms::wrap($content, $taxonomy, $before, $sep, $after) in /var/www/xxx/web/wp-content/plugins/front-end-editor/php/fields/post.php on line 0\nPHP message: PHP Warning: Declaration of FEE_Field_Category::wrap($content, $sep, $parents) should be compatible with FEE_Field_Terms::wrap($content, $taxonomy, $before, $sep, $after) in /var/www/xxx/web/wp-content/plugins/front-end-editor/php/fields/post.php on line 0", "event": { + "ingested": "2021-12-09T13:41:43.902420Z", + "original": "2019/10/30 23:26:34 [error] 205860#205860: *180289 FastCGI sent in stderr: \"PHP message: PHP Warning: Declaration of FEE_Field_Terms::wrap($content, $taxonomy, $before, $sep, $after) should be compatible with FEE_Field_Post::wrap($content, $post_id = 0) in /var/www/xxx/web/wp-content/plugins/front-end-editor/php/fields/post.php on line 0\nPHP message: PHP Warning: Declaration of FEE_Field_Tags::wrap($content, $before, $sep, $after) should be compatible with FEE_Field_Terms::wrap($content, $taxonomy, $before, $sep, $after) in /var/www/xxx/web/wp-content/plugins/front-end-editor/php/fields/post.php on line 0\nPHP message: PHP Warning: Declaration of FEE_Field_Category::wrap($content, $sep, $parents) should be compatible with FEE_Field_Terms::wrap($content, $taxonomy, $before, $sep, $after) in /var/www/xxx/web/wp-content/plugins/front-end-editor/php/fields/post.php on line 0", "category": [ "web" ], @@ -82,7 +102,11 @@ ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" - } + }, + "message": "FastCGI sent in stderr: \"PHP message: PHP Warning: Declaration of FEE_Field_Terms::wrap($content, $taxonomy, $before, $sep, $after) should be compatible with FEE_Field_Post::wrap($content, $post_id = 0) in /var/www/xxx/web/wp-content/plugins/front-end-editor/php/fields/post.php on line 0\nPHP message: PHP Warning: Declaration of FEE_Field_Tags::wrap($content, $before, $sep, $after) should be compatible with FEE_Field_Terms::wrap($content, $taxonomy, $before, $sep, $after) in /var/www/xxx/web/wp-content/plugins/front-end-editor/php/fields/post.php on line 0\nPHP message: PHP Warning: Declaration of FEE_Field_Category::wrap($content, $sep, $parents) should be compatible with FEE_Field_Terms::wrap($content, $taxonomy, $before, $sep, $after) in /var/www/xxx/web/wp-content/plugins/front-end-editor/php/fields/post.php on line 0", + "tags": [ + "preserve_original_event" + ] }, { "process": { @@ -92,6 +116,9 @@ } }, "@timestamp": "2019-11-05T14:50:44.000Z", + "ecs": { + "version": "1.12.0" + }, "nginx": { "error": { "connection_id": 3 @@ -100,8 +127,9 @@ "log": { "level": "error" }, - "message": "open() \"/usr/local/Cellar/nginx/1.10.2_1/html/adsasd\" failed (2: No such file or directory), client: 127.0.0.1, server: localhost, request: \"GET /pysio HTTP/1.1\", host: \"localhost:8080\"", "event": { + "ingested": "2021-12-09T13:41:43.902426Z", + "original": "2019/11/05 14:50:44 [error] 54053#0: *3 open() \"/usr/local/Cellar/nginx/1.10.2_1/html/adsasd\" failed (2: No such file or directory), client: 127.0.0.1, server: localhost, request: \"GET /pysio HTTP/1.1\", host: \"localhost:8080\"", "category": [ "web" ], @@ -110,7 +138,11 @@ ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" - } + }, + "message": "open() \"/usr/local/Cellar/nginx/1.10.2_1/html/adsasd\" failed (2: No such file or directory), client: 127.0.0.1, server: localhost, request: \"GET /pysio HTTP/1.1\", host: \"localhost:8080\"", + "tags": [ + "preserve_original_event" + ] } ] } \ No newline at end of file diff --git a/test/packages/nginx/data_stream/error/_dev/test/system/test-default-config.yml b/test/packages/nginx/data_stream/error/_dev/test/system/test-default-config.yml new file mode 100644 index 0000000000..ec7356ee93 --- /dev/null +++ b/test/packages/nginx/data_stream/error/_dev/test/system/test-default-config.yml @@ -0,0 +1,5 @@ +vars: ~ +data_stream: + vars: + paths: + - "{{SERVICE_LOGS_DIR}}/error.log*" diff --git a/test/packages/nginx/data_stream/error/agent/stream/httpjson.yml.hbs b/test/packages/nginx/data_stream/error/agent/stream/httpjson.yml.hbs new file mode 100644 index 0000000000..27d1775b51 --- /dev/null +++ b/test/packages/nginx/data_stream/error/agent/stream/httpjson.yml.hbs @@ -0,0 +1,63 @@ +config_version: 2 +interval: {{interval}} +{{#unless token}} +{{#if username}} +{{#if password}} +auth.basic.user: {{username}} +auth.basic.password: {{password}} +{{/if}} +{{/if}} +{{/unless}} +cursor: + index_earliest: + value: '[[.last_event.result.max_indextime]]' +request.url: {{url}}/services/search/jobs/export +{{#if ssl}} +request.ssl: {{ssl}} +{{/if}} +request.method: POST +request.transforms: + - set: + target: url.params.search + value: {{search}} | streamstats max(_indextime) AS max_indextime + - set: + target: url.params.output_mode + value: "json" + - set: + target: url.params.index_earliest + value: '[[ .cursor.index_earliest ]]' + default: '[[(now (parseDuration "-{{interval}}")).Unix]]' + - set: + target: url.params.index_latest + value: '[[(now).Unix]]' + - set: + target: header.Content-Type + value: application/x-www-form-urlencoded +{{#unless username}} +{{#unless password}} +{{#if token}} + - set: + target: header.Authorization + value: {{token}} +{{/if}} +{{/unless}} +{{/unless}} +response.decode_as: application/x-ndjson +response.split: + target: body.result._raw + type: string + delimiter: "\n" +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} \ No newline at end of file diff --git a/test/packages/nginx/data_stream/error/agent/stream/stream.yml.hbs b/test/packages/nginx/data_stream/error/agent/stream/stream.yml.hbs index 85ff82a8ec..e860976a65 100644 --- a/test/packages/nginx/data_stream/error/agent/stream/stream.yml.hbs +++ b/test/packages/nginx/data_stream/error/agent/stream/stream.yml.hbs @@ -2,10 +2,23 @@ paths: {{#each paths}} - {{this}} {{/each}} +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} exclude_files: [".gz$"] multiline: pattern: '^\d{4}\/\d{2}\/\d{2} ' negate: true match: after processors: +{{#if processors}} +{{processors}} +{{/if}} - add_locale: ~ diff --git a/test/packages/nginx/data_stream/error/elasticsearch/ingest_pipeline/default.yml b/test/packages/nginx/data_stream/error/elasticsearch/ingest_pipeline/default.yml index 96b46eb9c9..9ac85a34e9 100644 --- a/test/packages/nginx/data_stream/error/elasticsearch/ingest_pipeline/default.yml +++ b/test/packages/nginx/data_stream/error/elasticsearch/ingest_pipeline/default.yml @@ -1,52 +1,74 @@ --- description: Pipeline for parsing the Nginx error logs processors: -- grok: - field: message - patterns: - - '%{DATA:nginx.error.time} \[%{DATA:log.level}\] %{NUMBER:process.pid:long}#%{NUMBER:process.thread.id:long}: - (\*%{NUMBER:nginx.error.connection_id:long} )?%{GREEDYMULTILINE:message}' - pattern_definitions: - GREEDYMULTILINE: |- - (.| - | )* - ignore_missing: true -- rename: - field: '@timestamp' - target_field: event.created -- date: - if: ctx.event.timezone == null - field: nginx.error.time - target_field: '@timestamp' - formats: - - yyyy/MM/dd H:m:s - on_failure: - - append: - field: error.message - value: '{{ _ingest.on_failure_message }}' -- date: - if: ctx.event.timezone != null - field: nginx.error.time - target_field: '@timestamp' - formats: - - yyyy/MM/dd H:m:s - timezone: '{{ event.timezone }}' - on_failure: - - append: - field: error.message - value: '{{ _ingest.on_failure_message }}' -- remove: - field: nginx.error.time -- set: - field: event.kind - value: event -- append: - field: event.category - value: web -- append: - field: event.type - value: error + - pipeline: + if: ctx.message.startsWith('{') + name: '{{ IngestPipeline "third-party" }}' + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + - set: + field: ecs.version + value: '1.12.0' + - rename: + field: message + target_field: event.original + ignore_missing: true + - grok: + field: event.original + patterns: + - '%{DATA:nginx.error.time} \[%{DATA:log.level}\] %{NUMBER:process.pid:long}#%{NUMBER:process.thread.id:long}: + (\*%{NUMBER:nginx.error.connection_id:long} )?%{GREEDYMULTILINE:message}' + pattern_definitions: + GREEDYMULTILINE: |- + (.| + | )* + ignore_missing: true + - remove: + field: event.created + ignore_missing: true + ignore_failure: true + - rename: + field: '@timestamp' + target_field: event.created + - date: + if: ctx.event.timezone == null + field: nginx.error.time + target_field: '@timestamp' + formats: + - yyyy/MM/dd H:m:s + on_failure: + - append: + field: error.message + value: '{{ _ingest.on_failure_message }}' + - date: + if: ctx.event.timezone != null + field: nginx.error.time + target_field: '@timestamp' + formats: + - yyyy/MM/dd H:m:s + timezone: '{{ event.timezone }}' + on_failure: + - append: + field: error.message + value: '{{ _ingest.on_failure_message }}' + - remove: + field: nginx.error.time + - set: + field: event.kind + value: event + - append: + field: event.category + value: web + - append: + field: event.type + value: error + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: -- set: - field: error.message - value: '{{ _ingest.on_failure_message }}' + - set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/test/packages/nginx/data_stream/error/elasticsearch/ingest_pipeline/third-party.yml b/test/packages/nginx/data_stream/error/elasticsearch/ingest_pipeline/third-party.yml new file mode 100644 index 0000000000..da7fce1118 --- /dev/null +++ b/test/packages/nginx/data_stream/error/elasticsearch/ingest_pipeline/third-party.yml @@ -0,0 +1,42 @@ +--- +description: Pipeline for parsing nginx logs from third party api +processors: + - json: + field: message + target_field: json + - drop: + if: ctx.json?.result == null + - fingerprint: + fields: + - json.result._cd + - json.result._indextime + - json.result._raw + - json.result._time + - json.result.host + - json.result.source + target_field: '_id' + ignore_missing: true + - set: + copy_from: json.result._raw + field: message + ignore_empty_value: true + - set: + copy_from: json.result.host + field: host.name + ignore_empty_value: true + - set: + copy_from: json.result.source + field: file.path + ignore_empty_value: true + - remove: + field: + - json + ignore_missing: true +on_failure: + - append: + field: error.message + value: >- + error in third-party pipeline: + error in [{{_ingest.on_failure_processor_type}}] processor{{#_ingest.on_failure_processor_tag}} + with tag [{{_ingest.on_failure_processor_tag }}]{{/_ingest.on_failure_processor_tag}} + {{ _ingest.on_failure_message }} diff --git a/test/packages/nginx/data_stream/error/fields/agent.yml b/test/packages/nginx/data_stream/error/fields/agent.yml new file mode 100644 index 0000000000..e313ec8287 --- /dev/null +++ b/test/packages/nginx/data_stream/error/fields/agent.yml @@ -0,0 +1,204 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + +- name: input.type + type: keyword + description: Input type +- name: log.offset + type: long + description: Log offset diff --git a/test/packages/nginx/data_stream/error/fields/base-fields.yml b/test/packages/nginx/data_stream/error/fields/base-fields.yml index 885c2a63a9..7fc7303ce4 100644 --- a/test/packages/nginx/data_stream/error/fields/base-fields.yml +++ b/test/packages/nginx/data_stream/error/fields/base-fields.yml @@ -1,13 +1,20 @@ - name: data_stream.type - external: ecs + type: constant_keyword description: Data stream type. - value: logs - name: data_stream.dataset - external: ecs + type: constant_keyword description: Data stream dataset. - value: nginx.error - name: data_stream.namespace - external: ecs + type: constant_keyword + description: Data stream namespace. - name: '@timestamp' type: date description: Event timestamp. +- name: event.module + type: constant_keyword + description: Event module + value: nginx +- name: event.dataset + type: constant_keyword + description: Event dataset + value: nginx.error diff --git a/test/packages/nginx/data_stream/error/fields/ecs.yml b/test/packages/nginx/data_stream/error/fields/ecs.yml index bcfeec40fb..e26b3ec9bd 100644 --- a/test/packages/nginx/data_stream/error/fields/ecs.yml +++ b/test/packages/nginx/data_stream/error/fields/ecs.yml @@ -1,35 +1,16 @@ -- name: message - level: core - type: text - description: |- - For log events the message field contains the log message, optimized for viewing in a log viewer. - For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. - If multiple messages exist, they can be combined into one message. -- name: log - title: Log - group: 2 - type: group - fields: - - name: level - level: core - type: keyword - description: |- - Original log level of the log event. - If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity). - Some examples are `warn`, `err`, `i`, `informational`. - ignore_above: 1024 -- name: process - title: Process - group: 2 - type: group - fields: - - name: pid - level: core - type: long - format: string - description: Process id. - - name: thread.id - level: extended - type: long - format: string - description: Thread ID. +- external: ecs + name: ecs.version +- external: ecs + name: host.ip +- external: ecs + name: log.file.path +- external: ecs + name: log.level +- external: ecs + name: message +- external: ecs + name: process.pid +- external: ecs + name: process.thread.id +- external: ecs + name: tags diff --git a/test/packages/nginx/data_stream/error/manifest.yml b/test/packages/nginx/data_stream/error/manifest.yml index 961092483b..1a741682a6 100644 --- a/test/packages/nginx/data_stream/error/manifest.yml +++ b/test/packages/nginx/data_stream/error/manifest.yml @@ -1,6 +1,5 @@ title: Nginx error logs type: logs -release: experimental streams: - input: logfile vars: @@ -12,5 +11,73 @@ streams: show_user: true default: - /var/log/nginx/error.log* + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - nginx-error + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + title: Nginx error logs description: Collect Nginx error logs + - input: httpjson + title: Nginx error logs via Splunk REST API + description: Collect Nginx error logs via Splunk REST API + enabled: false + template_path: httpjson.yml.hbs + vars: + - name: interval + type: text + title: Interval to query REST API + description: Go Duration syntax (eg. 10s) + show_user: true + required: true + default: 10s + - name: search + type: text + title: Search String + show_user: true + required: true + default: search sourcetype=nginx:plus:error + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded + - nginx-error + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: >- + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. diff --git a/test/packages/nginx/data_stream/error/sample_event.json b/test/packages/nginx/data_stream/error/sample_event.json new file mode 100644 index 0000000000..b832f1b072 --- /dev/null +++ b/test/packages/nginx/data_stream/error/sample_event.json @@ -0,0 +1,77 @@ +{ + "agent": { + "hostname": "a73e7856c209", + "name": "a73e7856c209", + "id": "3987d2b3-b40a-4aa0-99fc-478f9d7079ea", + "ephemeral_id": "6d41da1c-5f71-4bd4-b326-a8913bfaa884", + "type": "filebeat", + "version": "7.11.0" + }, + "process": { + "pid": 1, + "thread": { + "id": 1 + } + }, + "nginx": { + "error": {} + }, + "log": { + "file": { + "path": "/tmp/service_logs/error.log" + }, + "offset": 0, + "level": "warn" + }, + "elastic_agent": { + "id": "5ca3af72-37c3-48b6-92e8-176d154bb66f", + "version": "7.11.0", + "snapshot": true + }, + "message": "conflicting server name \"localhost\" on 0.0.0.0:80, ignored", + "input": { + "type": "log" + }, + "@timestamp": "2020-12-03T11:44:39.000Z", + "ecs": { + "version": "1.6.0" + }, + "data_stream": { + "namespace": "ep", + "type": "logs", + "dataset": "nginx.error" + }, + "host": { + "hostname": "a73e7856c209", + "os": { + "kernel": "4.9.184-linuxkit", + "codename": "Core", + "name": "CentOS Linux", + "family": "redhat", + "version": "7 (Core)", + "platform": "centos" + }, + "containerized": true, + "ip": [ + "192.168.80.6" + ], + "name": "a73e7856c209", + "id": "06c26569966fd125c15acac5d7feffb6", + "mac": [ + "02:42:c0:a8:50:06" + ], + "architecture": "x86_64" + }, + "event": { + "timezone": "+00:00", + "created": "2020-12-03T11:44:52.803Z", + "kind": "event", + "category": [ + "web" + ], + "type": [ + "error" + ], + "dataset": "nginx.error" + } +} \ No newline at end of file diff --git a/test/packages/nginx/data_stream/ingress_controller/_dev/test/pipeline/test-ingest-raw.log b/test/packages/nginx/data_stream/ingress_controller/_dev/test/pipeline/test-ingest-raw.log deleted file mode 100644 index 8f65b971c3..0000000000 --- a/test/packages/nginx/data_stream/ingress_controller/_dev/test/pipeline/test-ingest-raw.log +++ /dev/null @@ -1,22 +0,0 @@ -192.168.64.1 - - [07/Feb/2020:11:48:51 +0000] "POST /products HTTP/1.1" 200 59 "-" "curl/7.54.0" 89 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.000 200 529a007902362a5f51385a5fa7049884 -192.168.64.1 - - [07/Feb/2020:11:49:15 +0000] "GET /products/42 HTTP/1.1" 200 59 "-" "curl/7.54.0" 91 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.000 200 68fa971ce4dfce685fdc01c877bfa645 -192.168.64.1 - - [07/Feb/2020:11:49:30 +0000] "DELETE /products/42 HTTP/1.1" 200 59 "-" "curl/7.54.0" 94 0.000 [default-web-8080] [] 172.17.0.5:8080 59 0.001 200 0be411044cb1cb67580e115413b2da60 -192.168.64.1 - - [07/Feb/2020:11:49:43 +0000] "PATCH /products/42 HTTP/1.1" 200 59 "-" "curl/7.54.0" 93 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.000 200 f479ab1d9cc8afcbac9e9f958ff8babc -192.168.64.1 - - [07/Feb/2020:11:49:50 +0000] "PATCHp /products/42 HTTP/1.1" 400 163 "-" "-" 0 0.000 [] [] - - - - 4c7d2079340e68353c7d0dfff00b904b -192.168.64.1 - - [07/Feb/2020:11:50:09 +0000] "geti /products/42 HTTP/1.1" 400 163 "-" "-" 0 0.000 [] [] - - - - efb0c5aa8be6cdeb4a7e7bd090e3d893 -192.168.64.1 - - [07/Feb/2020:11:55:05 +0000] "GET /products/42 HTTP/1.1" 200 59 "-" "Wget/1.20.3 (darwin18.6.0)" 157 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.000 200 457b71c3e1ee1887bb809effd301a0ec -192.168.64.1 - - [07/Feb/2020:11:55:57 +0000] "GET /products/42 HTTP/1.1" 200 59 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36" 450 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.000 200 da29abf31e4d6324cebe5e7bca370709 -192.168.64.1 - - [07/Feb/2020:11:55:57 +0000] "GET /favicon.ico HTTP/1.1" 200 59 "http://hello-world.info/products/42" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36" 381 0.000 [default-web-8080] [] 172.17.0.5:8080 59 0.000 200 e983c8cf3d713548baa50c9e2fffeb34 -192.168.64.1 - - [07/Feb/2020:11:56:24 +0000] "GET /v2 HTTP/1.1" 200 61 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36" 441 0.002 [default-web2-8080] [] 172.17.0.6:8080 61 0.001 200 3d7ff18ff4181a7db5013a76f975d900 -192.168.64.1 - - [07/Feb/2020:11:56:24 +0000] "GET /favicon.ico HTTP/1.1" 200 59 "http://hello-world.info/v2" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36" 372 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.002 200 d131fe4bcd359cf947f75efca4bfa553 -192.168.64.1 - - [07/Feb/2020:11:56:36 +0000] "GET /products/42 HTTP/1.1" 200 59 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15" 369 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.001 200 ef6629fcaaa1ea0d1a843cf2bf40571d -192.168.64.1 - - [07/Feb/2020:11:56:36 +0000] "GET /favicon.ico HTTP/1.1" 200 59 "http://hello-world.info/products/42" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15" 325 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.001 200 2593a1126588922449c183c8d9ddbbeb -192.168.64.1 - - [07/Feb/2020:11:56:54 +0000] "GET /products/42 HTTP/1.1" 200 59 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15" 369 0.002 [default-web-8080] [] 172.17.0.5:8080 59 0.002 200 0f76ea730f282d5759018eb756b23b14 -192.168.64.1 - - [07/Feb/2020:11:56:54 +0000] "GET / HTTP/1.1" 200 59 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15" 358 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.001 200 21efd18e3a7952fc78c0f2dcc1f05e69 -192.168.64.1 - - [07/Feb/2020:11:56:54 +0000] "GET /favicon.ico HTTP/1.1" 200 59 "http://hello-world.info/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15" 314 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.002 200 e096809c2cb46f004c3b538b23916e5b -192.168.64.1 - - [07/Feb/2020:11:56:56 +0000] "GET /v2 HTTP/1.1" 200 61 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15" 360 0.002 [default-web2-8080] [] 172.17.0.6:8080 61 0.002 200 0a2a92d080e664dd4e95c85d097c9d3d -192.168.64.1 - - [07/Feb/2020:11:56:56 +0000] "GET /favicon.ico HTTP/1.1" 200 59 "http://hello-world.info/v2" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15" 316 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.000 200 4c024ebfc20acfb2d59e542e3ed60789 -192.168.64.1 - - [07/Feb/2020:12:00:28 +0000] "GET /products/42?address=delhi+technological+university HTTP/1.1" 200 59 "-" "python-requests/2.22.0" 197 0.000 [default-web-8080] [] 172.17.0.5:8080 59 0.001 200 9a7babf34ca4ee59d90ac48d452a9214 -192.168.64.1 - - [07/Feb/2020:12:02:38 +0000] "GET /v2 HTTP/1.1" 200 61 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0" 343 0.000 [default-web2-8080] [] 172.17.0.6:8080 61 0.001 200 ba91c30454893c121879396b0a78be79 -192.168.64.1 - - [07/Feb/2020:12:02:38 +0000] "GET /favicon.ico HTTP/1.1" 200 59 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0" 262 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.000 200 98c81aa2d50c67f6fb1fa16d5ce62f8f -192.168.64.1 - - [07/Feb/2020:12:02:42 +0000] "GET /v2/some HTTP/1.1" 200 61 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0" 348 0.001 [default-web2-8080] [] 172.17.0.6:8080 61 0.000 200 835136ae24486dbb4156dcbe21f5d402 \ No newline at end of file diff --git a/test/packages/nginx/data_stream/ingress_controller/_dev/test/pipeline/test-ingest-raw.log-config.yml b/test/packages/nginx/data_stream/ingress_controller/_dev/test/pipeline/test-ingest-raw.log-config.yml deleted file mode 100644 index 958d74a23e..0000000000 --- a/test/packages/nginx/data_stream/ingress_controller/_dev/test/pipeline/test-ingest-raw.log-config.yml +++ /dev/null @@ -1,4 +0,0 @@ -multiline: - first_line_pattern: "^(?:[0-9]{1,3}\\.){3}[0-9]{1,3}" -fields: - "@timestamp": "2020-04-28T11:07:58.223Z" diff --git a/test/packages/nginx/data_stream/ingress_controller/_dev/test/pipeline/test-ingest-raw.log-expected.json b/test/packages/nginx/data_stream/ingress_controller/_dev/test/pipeline/test-ingest-raw.log-expected.json deleted file mode 100644 index 1cb6cd13c2..0000000000 --- a/test/packages/nginx/data_stream/ingress_controller/_dev/test/pipeline/test-ingest-raw.log-expected.json +++ /dev/null @@ -1,1479 +0,0 @@ -{ - "expected": [ - { - "@timestamp": "2020-02-07T11:48:51.000Z", - "nginx": { - "ingress_controller": { - "http": { - "request": { - "length": 89, - "id": "529a007902362a5f51385a5fa7049884", - "time": 0.001 - } - }, - "remote_ip_list": [ - "192.168.64.1" - ], - "upstream": { - "name": "default-web-8080", - "alternative_name": "", - "port": "8080", - "response": { - "length": 59, - "status_code": 200, - "time": 0.0 - }, - "ip": "172.17.0.5" - } - } - }, - "http": { - "request": { - "method": "post" - }, - "version": "1.1", - "response": { - "body": { - "bytes": 59 - }, - "status_code": 200 - } - }, - "source": { - "address": "" - }, - "event": { - "category": [ - "web" - ], - "type": [ - "info" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event", - "outcome": "success" - }, - "user_agent": { - "name": "curl", - "original": "curl/7.54.0", - "device": { - "name": "Other" - }, - "version": "7.54.0" - }, - "url": { - "original": "/products" - } - }, - { - "@timestamp": "2020-02-07T11:49:15.000Z", - "nginx": { - "ingress_controller": { - "http": { - "request": { - "length": 91, - "id": "68fa971ce4dfce685fdc01c877bfa645", - "time": 0.001 - } - }, - "remote_ip_list": [ - "192.168.64.1" - ], - "upstream": { - "name": "default-web-8080", - "alternative_name": "", - "port": "8080", - "response": { - "length": 59, - "status_code": 200, - "time": 0.0 - }, - "ip": "172.17.0.5" - } - } - }, - "http": { - "request": { - "method": "get" - }, - "version": "1.1", - "response": { - "body": { - "bytes": 59 - }, - "status_code": 200 - } - }, - "source": { - "address": "" - }, - "event": { - "category": [ - "web" - ], - "type": [ - "info" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event", - "outcome": "success" - }, - "user_agent": { - "name": "curl", - "original": "curl/7.54.0", - "device": { - "name": "Other" - }, - "version": "7.54.0" - }, - "url": { - "original": "/products/42" - } - }, - { - "@timestamp": "2020-02-07T11:49:30.000Z", - "nginx": { - "ingress_controller": { - "http": { - "request": { - "length": 94, - "id": "0be411044cb1cb67580e115413b2da60", - "time": 0.0 - } - }, - "remote_ip_list": [ - "192.168.64.1" - ], - "upstream": { - "name": "default-web-8080", - "alternative_name": "", - "port": "8080", - "response": { - "length": 59, - "status_code": 200, - "time": 0.001 - }, - "ip": "172.17.0.5" - } - } - }, - "http": { - "request": { - "method": "delete" - }, - "version": "1.1", - "response": { - "body": { - "bytes": 59 - }, - "status_code": 200 - } - }, - "source": { - "address": "" - }, - "event": { - "category": [ - "web" - ], - "type": [ - "info" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event", - "outcome": "success" - }, - "user_agent": { - "name": "curl", - "original": "curl/7.54.0", - "device": { - "name": "Other" - }, - "version": "7.54.0" - }, - "url": { - "original": "/products/42" - } - }, - { - "@timestamp": "2020-02-07T11:49:43.000Z", - "nginx": { - "ingress_controller": { - "http": { - "request": { - "length": 93, - "id": "f479ab1d9cc8afcbac9e9f958ff8babc", - "time": 0.001 - } - }, - "remote_ip_list": [ - "192.168.64.1" - ], - "upstream": { - "name": "default-web-8080", - "alternative_name": "", - "port": "8080", - "response": { - "length": 59, - "status_code": 200, - "time": 0.0 - }, - "ip": "172.17.0.5" - } - } - }, - "http": { - "request": { - "method": "patch" - }, - "version": "1.1", - "response": { - "body": { - "bytes": 59 - }, - "status_code": 200 - } - }, - "source": { - "address": "" - }, - "event": { - "category": [ - "web" - ], - "type": [ - "info" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event", - "outcome": "success" - }, - "user_agent": { - "name": "curl", - "original": "curl/7.54.0", - "device": { - "name": "Other" - }, - "version": "7.54.0" - }, - "url": { - "original": "/products/42" - } - }, - { - "@timestamp": "2020-02-07T11:49:50.000Z", - "nginx": { - "ingress_controller": { - "http": { - "request": { - "length": 0, - "id": "4c7d2079340e68353c7d0dfff00b904b", - "time": 0.0 - } - }, - "remote_ip_list": [ - "192.168.64.1" - ], - "upstream": { - "name": "", - "alternative_name": "" - } - } - }, - "http": { - "request": { - "method": "patchp" - }, - "version": "1.1", - "response": { - "body": { - "bytes": 163 - }, - "status_code": 400 - } - }, - "source": { - "address": "" - }, - "event": { - "category": [ - "web" - ], - "type": [ - "info" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event", - "outcome": "failure" - }, - "url": { - "original": "/products/42" - } - }, - { - "@timestamp": "2020-02-07T11:50:09.000Z", - "nginx": { - "ingress_controller": { - "http": { - "request": { - "length": 0, - "id": "efb0c5aa8be6cdeb4a7e7bd090e3d893", - "time": 0.0 - } - }, - "remote_ip_list": [ - "192.168.64.1" - ], - "upstream": { - "name": "", - "alternative_name": "" - } - } - }, - "http": { - "request": { - "method": "geti" - }, - "version": "1.1", - "response": { - "body": { - "bytes": 163 - }, - "status_code": 400 - } - }, - "source": { - "address": "" - }, - "event": { - "category": [ - "web" - ], - "type": [ - "info" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event", - "outcome": "failure" - }, - "url": { - "original": "/products/42" - } - }, - { - "@timestamp": "2020-02-07T11:55:05.000Z", - "nginx": { - "ingress_controller": { - "http": { - "request": { - "length": 157, - "id": "457b71c3e1ee1887bb809effd301a0ec", - "time": 0.001 - } - }, - "remote_ip_list": [ - "192.168.64.1" - ], - "upstream": { - "name": "default-web-8080", - "alternative_name": "", - "port": "8080", - "response": { - "length": 59, - "status_code": 200, - "time": 0.0 - }, - "ip": "172.17.0.5" - } - } - }, - "http": { - "request": { - "method": "get" - }, - "version": "1.1", - "response": { - "body": { - "bytes": 59 - }, - "status_code": 200 - } - }, - "source": { - "address": "" - }, - "event": { - "category": [ - "web" - ], - "type": [ - "info" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event", - "outcome": "success" - }, - "user_agent": { - "name": "Wget", - "original": "Wget/1.20.3 (darwin18.6.0)", - "device": { - "name": "Other" - }, - "version": "1.20.3" - }, - "url": { - "original": "/products/42" - } - }, - { - "@timestamp": "2020-02-07T11:55:57.000Z", - "nginx": { - "ingress_controller": { - "http": { - "request": { - "length": 450, - "id": "da29abf31e4d6324cebe5e7bca370709", - "time": 0.001 - } - }, - "remote_ip_list": [ - "192.168.64.1" - ], - "upstream": { - "name": "default-web-8080", - "alternative_name": "", - "port": "8080", - "response": { - "length": 59, - "status_code": 200, - "time": 0.0 - }, - "ip": "172.17.0.5" - } - } - }, - "http": { - "request": { - "method": "get" - }, - "version": "1.1", - "response": { - "body": { - "bytes": 59 - }, - "status_code": 200 - } - }, - "source": { - "address": "" - }, - "event": { - "category": [ - "web" - ], - "type": [ - "info" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event", - "outcome": "success" - }, - "user_agent": { - "name": "Chrome", - "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36", - "os": { - "name": "Mac OS X", - "version": "10.14.6", - "full": "Mac OS X 10.14.6" - }, - "device": { - "name": "Mac" - }, - "version": "79.0.3945.130" - }, - "url": { - "original": "/products/42" - } - }, - { - "@timestamp": "2020-02-07T11:55:57.000Z", - "nginx": { - "ingress_controller": { - "http": { - "request": { - "length": 381, - "id": "e983c8cf3d713548baa50c9e2fffeb34", - "time": 0.0 - } - }, - "remote_ip_list": [ - "192.168.64.1" - ], - "upstream": { - "name": "default-web-8080", - "alternative_name": "", - "port": "8080", - "response": { - "length": 59, - "status_code": 200, - "time": 0.0 - }, - "ip": "172.17.0.5" - } - } - }, - "http": { - "request": { - "method": "get", - "referrer": "http://hello-world.info/products/42" - }, - "version": "1.1", - "response": { - "body": { - "bytes": 59 - }, - "status_code": 200 - } - }, - "source": { - "address": "" - }, - "event": { - "category": [ - "web" - ], - "type": [ - "info" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event", - "outcome": "success" - }, - "user_agent": { - "name": "Chrome", - "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36", - "os": { - "name": "Mac OS X", - "version": "10.14.6", - "full": "Mac OS X 10.14.6" - }, - "device": { - "name": "Mac" - }, - "version": "79.0.3945.130" - }, - "url": { - "original": "/favicon.ico" - } - }, - { - "@timestamp": "2020-02-07T11:56:24.000Z", - "nginx": { - "ingress_controller": { - "http": { - "request": { - "length": 441, - "id": "3d7ff18ff4181a7db5013a76f975d900", - "time": 0.002 - } - }, - "remote_ip_list": [ - "192.168.64.1" - ], - "upstream": { - "name": "default-web2-8080", - "alternative_name": "", - "port": "8080", - "response": { - "length": 61, - "status_code": 200, - "time": 0.001 - }, - "ip": "172.17.0.6" - } - } - }, - "http": { - "request": { - "method": "get" - }, - "version": "1.1", - "response": { - "body": { - "bytes": 61 - }, - "status_code": 200 - } - }, - "source": { - "address": "" - }, - "event": { - "category": [ - "web" - ], - "type": [ - "info" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event", - "outcome": "success" - }, - "user_agent": { - "name": "Chrome", - "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36", - "os": { - "name": "Mac OS X", - "version": "10.14.6", - "full": "Mac OS X 10.14.6" - }, - "device": { - "name": "Mac" - }, - "version": "79.0.3945.130" - }, - "url": { - "original": "/v2" - } - }, - { - "@timestamp": "2020-02-07T11:56:24.000Z", - "nginx": { - "ingress_controller": { - "http": { - "request": { - "length": 372, - "id": "d131fe4bcd359cf947f75efca4bfa553", - "time": 0.001 - } - }, - "remote_ip_list": [ - "192.168.64.1" - ], - "upstream": { - "name": "default-web-8080", - "alternative_name": "", - "port": "8080", - "response": { - "length": 59, - "status_code": 200, - "time": 0.002 - }, - "ip": "172.17.0.5" - } - } - }, - "http": { - "request": { - "method": "get", - "referrer": "http://hello-world.info/v2" - }, - "version": "1.1", - "response": { - "body": { - "bytes": 59 - }, - "status_code": 200 - } - }, - "source": { - "address": "" - }, - "event": { - "category": [ - "web" - ], - "type": [ - "info" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event", - "outcome": "success" - }, - "user_agent": { - "name": "Chrome", - "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36", - "os": { - "name": "Mac OS X", - "version": "10.14.6", - "full": "Mac OS X 10.14.6" - }, - "device": { - "name": "Mac" - }, - "version": "79.0.3945.130" - }, - "url": { - "original": "/favicon.ico" - } - }, - { - "@timestamp": "2020-02-07T11:56:36.000Z", - "nginx": { - "ingress_controller": { - "http": { - "request": { - "length": 369, - "id": "ef6629fcaaa1ea0d1a843cf2bf40571d", - "time": 0.001 - } - }, - "remote_ip_list": [ - "192.168.64.1" - ], - "upstream": { - "name": "default-web-8080", - "alternative_name": "", - "port": "8080", - "response": { - "length": 59, - "status_code": 200, - "time": 0.001 - }, - "ip": "172.17.0.5" - } - } - }, - "http": { - "request": { - "method": "get" - }, - "version": "1.1", - "response": { - "body": { - "bytes": 59 - }, - "status_code": 200 - } - }, - "source": { - "address": "" - }, - "event": { - "category": [ - "web" - ], - "type": [ - "info" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event", - "outcome": "success" - }, - "user_agent": { - "name": "Safari", - "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", - "os": { - "name": "Mac OS X", - "version": "10.14.6", - "full": "Mac OS X 10.14.6" - }, - "device": { - "name": "Mac" - }, - "version": "13.0.5" - }, - "url": { - "original": "/products/42" - } - }, - { - "@timestamp": "2020-02-07T11:56:36.000Z", - "nginx": { - "ingress_controller": { - "http": { - "request": { - "length": 325, - "id": "2593a1126588922449c183c8d9ddbbeb", - "time": 0.001 - } - }, - "remote_ip_list": [ - "192.168.64.1" - ], - "upstream": { - "name": "default-web-8080", - "alternative_name": "", - "port": "8080", - "response": { - "length": 59, - "status_code": 200, - "time": 0.001 - }, - "ip": "172.17.0.5" - } - } - }, - "http": { - "request": { - "method": "get", - "referrer": "http://hello-world.info/products/42" - }, - "version": "1.1", - "response": { - "body": { - "bytes": 59 - }, - "status_code": 200 - } - }, - "source": { - "address": "" - }, - "event": { - "category": [ - "web" - ], - "type": [ - "info" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event", - "outcome": "success" - }, - "user_agent": { - "name": "Safari", - "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", - "os": { - "name": "Mac OS X", - "version": "10.14.6", - "full": "Mac OS X 10.14.6" - }, - "device": { - "name": "Mac" - }, - "version": "13.0.5" - }, - "url": { - "original": "/favicon.ico" - } - }, - { - "@timestamp": "2020-02-07T11:56:54.000Z", - "nginx": { - "ingress_controller": { - "http": { - "request": { - "length": 369, - "id": "0f76ea730f282d5759018eb756b23b14", - "time": 0.002 - } - }, - "remote_ip_list": [ - "192.168.64.1" - ], - "upstream": { - "name": "default-web-8080", - "alternative_name": "", - "port": "8080", - "response": { - "length": 59, - "status_code": 200, - "time": 0.002 - }, - "ip": "172.17.0.5" - } - } - }, - "http": { - "request": { - "method": "get" - }, - "version": "1.1", - "response": { - "body": { - "bytes": 59 - }, - "status_code": 200 - } - }, - "source": { - "address": "" - }, - "event": { - "category": [ - "web" - ], - "type": [ - "info" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event", - "outcome": "success" - }, - "user_agent": { - "name": "Safari", - "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", - "os": { - "name": "Mac OS X", - "version": "10.14.6", - "full": "Mac OS X 10.14.6" - }, - "device": { - "name": "Mac" - }, - "version": "13.0.5" - }, - "url": { - "original": "/products/42" - } - }, - { - "@timestamp": "2020-02-07T11:56:54.000Z", - "nginx": { - "ingress_controller": { - "http": { - "request": { - "length": 358, - "id": "21efd18e3a7952fc78c0f2dcc1f05e69", - "time": 0.001 - } - }, - "remote_ip_list": [ - "192.168.64.1" - ], - "upstream": { - "name": "default-web-8080", - "alternative_name": "", - "port": "8080", - "response": { - "length": 59, - "status_code": 200, - "time": 0.001 - }, - "ip": "172.17.0.5" - } - } - }, - "http": { - "request": { - "method": "get" - }, - "version": "1.1", - "response": { - "body": { - "bytes": 59 - }, - "status_code": 200 - } - }, - "source": { - "address": "" - }, - "event": { - "category": [ - "web" - ], - "type": [ - "info" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event", - "outcome": "success" - }, - "user_agent": { - "name": "Safari", - "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", - "os": { - "name": "Mac OS X", - "version": "10.14.6", - "full": "Mac OS X 10.14.6" - }, - "device": { - "name": "Mac" - }, - "version": "13.0.5" - }, - "url": { - "original": "/" - } - }, - { - "@timestamp": "2020-02-07T11:56:54.000Z", - "nginx": { - "ingress_controller": { - "http": { - "request": { - "length": 314, - "id": "e096809c2cb46f004c3b538b23916e5b", - "time": 0.001 - } - }, - "remote_ip_list": [ - "192.168.64.1" - ], - "upstream": { - "name": "default-web-8080", - "alternative_name": "", - "port": "8080", - "response": { - "length": 59, - "status_code": 200, - "time": 0.002 - }, - "ip": "172.17.0.5" - } - } - }, - "http": { - "request": { - "method": "get", - "referrer": "http://hello-world.info/" - }, - "version": "1.1", - "response": { - "body": { - "bytes": 59 - }, - "status_code": 200 - } - }, - "source": { - "address": "" - }, - "event": { - "category": [ - "web" - ], - "type": [ - "info" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event", - "outcome": "success" - }, - "user_agent": { - "name": "Safari", - "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", - "os": { - "name": "Mac OS X", - "version": "10.14.6", - "full": "Mac OS X 10.14.6" - }, - "device": { - "name": "Mac" - }, - "version": "13.0.5" - }, - "url": { - "original": "/favicon.ico" - } - }, - { - "@timestamp": "2020-02-07T11:56:56.000Z", - "nginx": { - "ingress_controller": { - "http": { - "request": { - "length": 360, - "id": "0a2a92d080e664dd4e95c85d097c9d3d", - "time": 0.002 - } - }, - "remote_ip_list": [ - "192.168.64.1" - ], - "upstream": { - "name": "default-web2-8080", - "alternative_name": "", - "port": "8080", - "response": { - "length": 61, - "status_code": 200, - "time": 0.002 - }, - "ip": "172.17.0.6" - } - } - }, - "http": { - "request": { - "method": "get" - }, - "version": "1.1", - "response": { - "body": { - "bytes": 61 - }, - "status_code": 200 - } - }, - "source": { - "address": "" - }, - "event": { - "category": [ - "web" - ], - "type": [ - "info" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event", - "outcome": "success" - }, - "user_agent": { - "name": "Safari", - "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", - "os": { - "name": "Mac OS X", - "version": "10.14.6", - "full": "Mac OS X 10.14.6" - }, - "device": { - "name": "Mac" - }, - "version": "13.0.5" - }, - "url": { - "original": "/v2" - } - }, - { - "@timestamp": "2020-02-07T11:56:56.000Z", - "nginx": { - "ingress_controller": { - "http": { - "request": { - "length": 316, - "id": "4c024ebfc20acfb2d59e542e3ed60789", - "time": 0.001 - } - }, - "remote_ip_list": [ - "192.168.64.1" - ], - "upstream": { - "name": "default-web-8080", - "alternative_name": "", - "port": "8080", - "response": { - "length": 59, - "status_code": 200, - "time": 0.0 - }, - "ip": "172.17.0.5" - } - } - }, - "http": { - "request": { - "method": "get", - "referrer": "http://hello-world.info/v2" - }, - "version": "1.1", - "response": { - "body": { - "bytes": 59 - }, - "status_code": 200 - } - }, - "source": { - "address": "" - }, - "event": { - "category": [ - "web" - ], - "type": [ - "info" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event", - "outcome": "success" - }, - "user_agent": { - "name": "Safari", - "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", - "os": { - "name": "Mac OS X", - "version": "10.14.6", - "full": "Mac OS X 10.14.6" - }, - "device": { - "name": "Mac" - }, - "version": "13.0.5" - }, - "url": { - "original": "/favicon.ico" - } - }, - { - "@timestamp": "2020-02-07T12:00:28.000Z", - "nginx": { - "ingress_controller": { - "http": { - "request": { - "length": 197, - "id": "9a7babf34ca4ee59d90ac48d452a9214", - "time": 0.0 - } - }, - "remote_ip_list": [ - "192.168.64.1" - ], - "upstream": { - "name": "default-web-8080", - "alternative_name": "", - "port": "8080", - "response": { - "length": 59, - "status_code": 200, - "time": 0.001 - }, - "ip": "172.17.0.5" - } - } - }, - "http": { - "request": { - "method": "get" - }, - "version": "1.1", - "response": { - "body": { - "bytes": 59 - }, - "status_code": 200 - } - }, - "source": { - "address": "" - }, - "event": { - "category": [ - "web" - ], - "type": [ - "info" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event", - "outcome": "success" - }, - "user_agent": { - "name": "Python Requests", - "original": "python-requests/2.22.0", - "device": { - "name": "Other" - }, - "version": "2.22" - }, - "url": { - "original": "/products/42?address=delhi+technological+university" - } - }, - { - "@timestamp": "2020-02-07T12:02:38.000Z", - "nginx": { - "ingress_controller": { - "http": { - "request": { - "length": 343, - "id": "ba91c30454893c121879396b0a78be79", - "time": 0.0 - } - }, - "remote_ip_list": [ - "192.168.64.1" - ], - "upstream": { - "name": "default-web2-8080", - "alternative_name": "", - "port": "8080", - "response": { - "length": 61, - "status_code": 200, - "time": 0.001 - }, - "ip": "172.17.0.6" - } - } - }, - "http": { - "request": { - "method": "get" - }, - "version": "1.1", - "response": { - "body": { - "bytes": 61 - }, - "status_code": 200 - } - }, - "source": { - "address": "" - }, - "event": { - "category": [ - "web" - ], - "type": [ - "info" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event", - "outcome": "success" - }, - "user_agent": { - "name": "Firefox", - "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0", - "os": { - "name": "Mac OS X", - "version": "10.14", - "full": "Mac OS X 10.14" - }, - "device": { - "name": "Mac" - }, - "version": "72.0." - }, - "url": { - "original": "/v2" - } - }, - { - "@timestamp": "2020-02-07T12:02:38.000Z", - "nginx": { - "ingress_controller": { - "http": { - "request": { - "length": 262, - "id": "98c81aa2d50c67f6fb1fa16d5ce62f8f", - "time": 0.001 - } - }, - "remote_ip_list": [ - "192.168.64.1" - ], - "upstream": { - "name": "default-web-8080", - "alternative_name": "", - "port": "8080", - "response": { - "length": 59, - "status_code": 200, - "time": 0.0 - }, - "ip": "172.17.0.5" - } - } - }, - "http": { - "request": { - "method": "get" - }, - "version": "1.1", - "response": { - "body": { - "bytes": 59 - }, - "status_code": 200 - } - }, - "source": { - "address": "" - }, - "event": { - "category": [ - "web" - ], - "type": [ - "info" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event", - "outcome": "success" - }, - "user_agent": { - "name": "Firefox", - "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0", - "os": { - "name": "Mac OS X", - "version": "10.14", - "full": "Mac OS X 10.14" - }, - "device": { - "name": "Mac" - }, - "version": "72.0." - }, - "url": { - "original": "/favicon.ico" - } - }, - { - "@timestamp": "2020-02-07T12:02:42.000Z", - "nginx": { - "ingress_controller": { - "http": { - "request": { - "length": 348, - "id": "835136ae24486dbb4156dcbe21f5d402", - "time": 0.001 - } - }, - "remote_ip_list": [ - "192.168.64.1" - ], - "upstream": { - "name": "default-web2-8080", - "alternative_name": "", - "port": "8080", - "response": { - "length": 61, - "status_code": 200, - "time": 0.0 - }, - "ip": "172.17.0.6" - } - } - }, - "http": { - "request": { - "method": "get" - }, - "version": "1.1", - "response": { - "body": { - "bytes": 61 - }, - "status_code": 200 - } - }, - "source": { - "address": "" - }, - "event": { - "category": [ - "web" - ], - "type": [ - "info" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event", - "outcome": "success" - }, - "user_agent": { - "name": "Firefox", - "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0", - "os": { - "name": "Mac OS X", - "version": "10.14", - "full": "Mac OS X 10.14" - }, - "device": { - "name": "Mac" - }, - "version": "72.0." - }, - "url": { - "original": "/v2/some" - } - } - ] -} \ No newline at end of file diff --git a/test/packages/nginx/data_stream/ingress_controller/agent/stream/stream.yml.hbs b/test/packages/nginx/data_stream/ingress_controller/agent/stream/stream.yml.hbs deleted file mode 100644 index 8a1b413517..0000000000 --- a/test/packages/nginx/data_stream/ingress_controller/agent/stream/stream.yml.hbs +++ /dev/null @@ -1,7 +0,0 @@ -paths: -{{#each paths}} - - {{this}} -{{/each}} -exclude_files: [".gz$"] -processors: - - add_locale: ~ diff --git a/test/packages/nginx/data_stream/ingress_controller/elasticsearch/ingest_pipeline/default.yml b/test/packages/nginx/data_stream/ingress_controller/elasticsearch/ingest_pipeline/default.yml deleted file mode 100644 index abf54fcd2c..0000000000 --- a/test/packages/nginx/data_stream/ingress_controller/elasticsearch/ingest_pipeline/default.yml +++ /dev/null @@ -1,173 +0,0 @@ ---- -description: Pipeline for parsing Nginx ingress controller access logs. Requires the - geoip and user_agent plugins. -processors: -- grok: - field: message - patterns: - - (%{NGINX_HOST} )?"?(?:%{NGINX_ADDRESS_LIST:nginx.ingress_controller.remote_ip_list}|%{NOTSPACE:source.address}) - - (-|%{DATA:user.name}) \[%{HTTPDATE:nginx.ingress_controller.time}\] "%{DATA:nginx.ingress_controller.info}" - %{NUMBER:http.response.status_code:long} %{NUMBER:http.response.body.bytes:long} - "(-|%{DATA:http.request.referrer})" "(-|%{DATA:user_agent.original})" %{NUMBER:nginx.ingress_controller.http.request.length:long} - %{NUMBER:nginx.ingress_controller.http.request.time:double} \[%{DATA:nginx.ingress_controller.upstream.name}\] - \[%{DATA:nginx.ingress_controller.upstream.alternative_name}\] (%{UPSTREAM_ADDRESS}|-) - (%{NUMBER:nginx.ingress_controller.upstream.response.length:long}|-) (%{NUMBER:nginx.ingress_controller.upstream.response.time:double}|-) - (%{NUMBER:nginx.ingress_controller.upstream.response.status_code:long}|-) %{GREEDYDATA:nginx.ingress_controller.http.request.id} - pattern_definitions: - NGINX_HOST: (?:%{IP:destination.ip}|%{NGINX_NOTSEPARATOR:destination.domain})(:%{NUMBER:destination.port})? - NGINX_NOTSEPARATOR: "[^\t ,:]+" - NGINX_ADDRESS_LIST: (?:%{IP}|%{WORD})("?,?\s*(?:%{IP}|%{WORD}))* - UPSTREAM_ADDRESS: '%{IP:nginx.ingress_controller.upstream.ip}(:%{NUMBER:nginx.ingress_controller.upstream.port})?' - ignore_missing: true -- grok: - field: nginx.ingress_controller.info - patterns: - - '%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}' - - "" - ignore_missing: true -- remove: - field: nginx.ingress_controller.info -- split: - field: nginx.ingress_controller.remote_ip_list - separator: '"?,?\s+' - ignore_missing: true -- split: - field: nginx.ingress_controller.origin - separator: '"?,?\s+' - ignore_missing: true -- set: - field: source.address - if: ctx.source?.address == null - value: "" -- script: - if: ctx.nginx?.access?.remote_ip_list != null && ctx.nginx.ingress_controller.remote_ip_list.length > 0 - lang: painless - source: >- - boolean isPrivate(def dot, def ip) { - try { - StringTokenizer tok = new StringTokenizer(ip, dot); - int firstByte = Integer.parseInt(tok.nextToken()); - int secondByte = Integer.parseInt(tok.nextToken()); - if (firstByte == 10) { - return true; - } - if (firstByte == 192 && secondByte == 168) { - return true; - } - if (firstByte == 172 && secondByte >= 16 && secondByte <= 31) { - return true; - } - if (firstByte == 127) { - return true; - } - return false; - } - catch (Exception e) { - return false; - } - } - try { - ctx.source.address = null; - if (ctx.nginx.ingress_controller.remote_ip_list == null) { - return; - } - def found = false; - for (def item : ctx.nginx.ingress_controller.remote_ip_list) { - if (!isPrivate(params.dot, item)) { - ctx.source.address = item; - found = true; - break; - } - } - if (!found) { - ctx.source.address = ctx.nginx.ingress_controller.remote_ip_list[0]; - } - } - catch (Exception e) { - ctx.source.address = null; - } - params: - dot: . -- remove: - field: source.address - if: ctx.source.address == null -- grok: - field: source.address - patterns: - - ^%{IP:source.ip}$ - ignore_failure: true -- remove: - field: message -- rename: - field: '@timestamp' - target_field: event.created -- date: - field: nginx.ingress_controller.time - target_field: '@timestamp' - formats: - - dd/MMM/yyyy:H:m:s Z - on_failure: - - append: - field: error.message - value: '{{ _ingest.on_failure_message }}' -- remove: - field: nginx.ingress_controller.time -- user_agent: - field: user_agent.original - ignore_missing: true -- geoip: - field: source.ip - target_field: source.geo - ignore_missing: true -- geoip: - database_file: GeoLite2-ASN.mmdb - field: source.ip - target_field: source.as - properties: - - asn - - organization_name - ignore_missing: true -- rename: - field: source.as.asn - target_field: source.as.number - ignore_missing: true -- rename: - field: source.as.organization_name - target_field: source.as.organization.name - ignore_missing: true -- set: - field: event.kind - value: event -- append: - field: event.category - value: web -- append: - field: event.type - value: info -- set: - field: event.outcome - value: success - if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400" -- set: - field: event.outcome - value: failure - if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400" -- lowercase: - field: http.request.method - ignore_missing: true -- append: - field: related.ip - value: "{{source.ip}}" - if: "ctx?.source?.ip != null" -- append: - field: related.ip - value: "{{destination.ip}}" - if: "ctx?.destination?.ip != null" -- append: - field: related.user - value: "{{user.name}}" - if: "ctx?.user?.name != null" -on_failure: -- set: - field: error.message - value: '{{ _ingest.on_failure_message }}' diff --git a/test/packages/nginx/data_stream/ingress_controller/fields/ecs.yml b/test/packages/nginx/data_stream/ingress_controller/fields/ecs.yml deleted file mode 100644 index 2710ddc7f8..0000000000 --- a/test/packages/nginx/data_stream/ingress_controller/fields/ecs.yml +++ /dev/null @@ -1,142 +0,0 @@ -- name: http - title: HTTP - group: 2 - type: group - fields: - - name: request.method - level: extended - type: keyword - description: |- - HTTP request method. - The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". - ignore_above: 1024 - - name: request.referrer - level: extended - type: keyword - description: Referrer for this HTTP request. - ignore_above: 1024 - - name: response.body.bytes - level: extended - type: long - format: bytes - description: Size in bytes of the response body. - - name: response.status_code - level: extended - type: long - format: string - description: HTTP response status code. - - name: version - level: extended - type: keyword - description: HTTP version. - ignore_above: 1024 -- name: source - title: Source - group: 2 - type: group - fields: - - name: address - type: keyword - description: Source address - - name: geo.city_name - level: core - type: keyword - description: City name. - ignore_above: 1024 - - name: geo.continent_name - level: core - type: keyword - description: Name of the continent. - ignore_above: 1024 - - name: geo.country_iso_code - level: core - type: keyword - description: Country ISO code. - ignore_above: 1024 - - name: geo.location - level: core - type: geo_point - description: Longitude and latitude. - - name: geo.region_iso_code - level: core - type: keyword - description: Region ISO code. - ignore_above: 1024 - - name: geo.region_name - level: core - type: keyword - description: Region name. - ignore_above: 1024 -- name: url - title: URL - group: 2 - type: group - fields: - - name: original - level: extended - type: keyword - description: |- - Unmodified original url as seen in the event source. - Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. - This field is meant to represent the URL as it was observed, complete or not. - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false -- name: user - title: User - group: 2 - type: group - fields: - - name: name - level: core - type: keyword - description: Short name or login of the user. - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false -- name: user_agent - title: User agent - group: 2 - type: group - fields: - - name: device.name - level: extended - type: keyword - description: Name of the device. - ignore_above: 1024 - - name: name - level: extended - type: keyword - description: Name of the user agent. - ignore_above: 1024 - - name: original - level: extended - type: keyword - description: Unparsed user_agent string. - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - - name: os.name - level: extended - type: keyword - description: Operating system name, without the version. - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false -- name: user_agent.os.version - type: keyword -- name: user_agent.os.full - type: keyword -- name: user_agent.version - type: keyword diff --git a/test/packages/nginx/data_stream/ingress_controller/fields/fields.yml b/test/packages/nginx/data_stream/ingress_controller/fields/fields.yml deleted file mode 100644 index e6ffd2c173..0000000000 --- a/test/packages/nginx/data_stream/ingress_controller/fields/fields.yml +++ /dev/null @@ -1,58 +0,0 @@ -- name: nginx.ingress_controller - type: group - fields: - - name: remote_ip_list - type: keyword - description: | - An array of remote IP addresses. It is a list because it is common to include, besides the client IP address, IP addresses from headers like `X-Forwarded-For`. Real source IP is restored to `source.ip`. - - name: http.request.length - type: long - format: bytes - description: | - The request length (including request line, header, and request body) - - name: http.request.time - type: double - format: duration - description: | - Time elapsed since the first bytes were read from the client - - name: upstream.name - type: text - description: | - The name of the upstream. - - name: upstream.alternative_name - type: text - description: | - The name of the alternative upstream. - - name: upstream.response.length - type: long - format: bytes - description: | - The length of the response obtained from the upstream server - - name: upstream.response.time - type: double - format: duration - description: | - The time spent on receiving the response from the upstream server as seconds with millisecond resolution - - name: upstream.response.status_code - type: long - description: | - The status code of the response obtained from the upstream server - - name: http.request.id - type: text - description: | - The randomly generated ID of the request - - name: upstream.ip - type: ip - description: | - The IP address of the upstream server. If several servers were contacted during request processing, their addresses are separated by commas. - - name: upstream.port - type: keyword - description: | - The port of the upstream server. - - name: user_agent - type: group - - name: geoip - type: group -- name: event.created - type: date - description: Date/time when the event was first read by an agent, or by your pipeline. diff --git a/test/packages/nginx/data_stream/ingress_controller/manifest.yml b/test/packages/nginx/data_stream/ingress_controller/manifest.yml deleted file mode 100644 index 64dfb7fce3..0000000000 --- a/test/packages/nginx/data_stream/ingress_controller/manifest.yml +++ /dev/null @@ -1,17 +0,0 @@ -title: Nginx ingress_controller logs -type: logs -release: experimental -streams: - - input: logfile - enabled: false - vars: - - name: paths - type: text - title: Paths - multi: true - required: true - show_user: true - default: - - /var/log/nginx/ingress.log* - title: Nginx ingress controller logs - description: Collect Nginx ingress controller logs diff --git a/test/packages/nginx/data_stream/stubstatus/_dev/test/system/test-default-config.yml b/test/packages/nginx/data_stream/stubstatus/_dev/test/system/test-default-config.yml new file mode 100644 index 0000000000..0e5231dc8b --- /dev/null +++ b/test/packages/nginx/data_stream/stubstatus/_dev/test/system/test-default-config.yml @@ -0,0 +1,6 @@ +vars: + hosts: + - http://{{Hostname}}:{{Port}} +data_stream: + vars: + server_status_path: /server-status diff --git a/test/packages/nginx/data_stream/stubstatus/fields/agent.yml b/test/packages/nginx/data_stream/stubstatus/fields/agent.yml new file mode 100644 index 0000000000..da4e652c53 --- /dev/null +++ b/test/packages/nginx/data_stream/stubstatus/fields/agent.yml @@ -0,0 +1,198 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + diff --git a/test/packages/nginx/data_stream/stubstatus/fields/base-fields.yml b/test/packages/nginx/data_stream/stubstatus/fields/base-fields.yml index 40db72e64f..399cecda4c 100644 --- a/test/packages/nginx/data_stream/stubstatus/fields/base-fields.yml +++ b/test/packages/nginx/data_stream/stubstatus/fields/base-fields.yml @@ -1,13 +1,20 @@ - name: data_stream.type - external: ecs + type: constant_keyword description: Data stream type. - value: metrics - name: data_stream.dataset - external: ecs + type: constant_keyword description: Data stream dataset. - value: nginx.stubstatus - name: data_stream.namespace - external: ecs + type: constant_keyword + description: Data stream namespace. - name: '@timestamp' type: date description: Event timestamp. +- name: event.module + type: constant_keyword + description: Event module + value: nginx +- name: event.dataset + type: constant_keyword + description: Event dataset + value: nginx.stubstatus diff --git a/test/packages/nginx/data_stream/stubstatus/fields/ecs.yml b/test/packages/nginx/data_stream/stubstatus/fields/ecs.yml new file mode 100644 index 0000000000..aae79b4b9f --- /dev/null +++ b/test/packages/nginx/data_stream/stubstatus/fields/ecs.yml @@ -0,0 +1,8 @@ +- external: ecs + name: host.ip +- external: ecs + name: ecs.version +- external: ecs + name: service.address +- external: ecs + name: service.type diff --git a/test/packages/nginx/data_stream/stubstatus/fields/fields.yml b/test/packages/nginx/data_stream/stubstatus/fields/fields.yml index a36adb0e12..5ded312df6 100644 --- a/test/packages/nginx/data_stream/stubstatus/fields/fields.yml +++ b/test/packages/nginx/data_stream/stubstatus/fields/fields.yml @@ -41,9 +41,3 @@ type: long description: | The current number of idle client connections waiting for a request. -- name: ecs.version - external: ecs -- name: service.address - type: keyword -- name: service.type - external: ecs diff --git a/test/packages/nginx/data_stream/stubstatus/manifest.yml b/test/packages/nginx/data_stream/stubstatus/manifest.yml index 83e447dd95..53d1ec8cfd 100644 --- a/test/packages/nginx/data_stream/stubstatus/manifest.yml +++ b/test/packages/nginx/data_stream/stubstatus/manifest.yml @@ -1,6 +1,5 @@ title: Nginx stubstatus metrics type: metrics -release: experimental streams: - input: nginx/metrics vars: diff --git a/test/packages/nginx/data_stream/stubstatus/sample_event.json b/test/packages/nginx/data_stream/stubstatus/sample_event.json index 27aded9671..2f8a9bd9bf 100644 --- a/test/packages/nginx/data_stream/stubstatus/sample_event.json +++ b/test/packages/nginx/data_stream/stubstatus/sample_event.json @@ -1,40 +1,72 @@ { - "@timestamp": "2020-04-28T11:07:58.223Z", + "@timestamp": "2020-12-03T11:47:31.996Z", + "host": { + "hostname": "a73e7856c209", + "architecture": "x86_64", + "os": { + "codename": "Core", + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.9.184-linuxkit" + }, + "name": "a73e7856c209", + "id": "06c26569966fd125c15acac5d7feffb6", + "containerized": true, + "ip": [ + "192.168.80.6" + ], + "mac": [ + "02:42:c0:a8:50:06" + ] + }, "service": { "type": "nginx", - "address": "127.0.0.1:8081" + "address": "http://elastic-package-service_nginx_1:80/server-status" }, "nginx": { "stubstatus": { + "requests": 13, "waiting": 0, - "hostname": "127.0.0.1:8081", + "hostname": "elastic-package-service_nginx_1:80", + "accepts": 13, + "handled": 13, + "current": 13, "dropped": 0, "writing": 1, - "handled": 7339, - "requests": 7411, - "reading": 0, - "accepts": 7339, - "current": 10, - "active": 1 + "active": 1, + "reading": 0 } }, - "ecs": { - "version": "1.5.0" + "elastic_agent": { + "snapshot": true, + "version": "7.11.0", + "id": "5ca3af72-37c3-48b6-92e8-176d154bb66f" }, - "agent": { - "type": "metricbeat", - "ephemeral_id": "8eb07b4f-df58-4794-8e00-60f1443f33b6", - "hostname": "MacBook-Elastic.local", - "id": "e47f6e4d-5277-46f3-801d-221c7584c604", - "version": "8.0.0" + "ecs": { + "version": "1.6.0" }, "event": { + "dataset": "nginx.stubstatus", "module": "nginx", - "duration": 1112095, - "dataset": "nginx.stubstatus" + "duration": 2231100 }, "metricset": { "period": 10000, "name": "stubstatus" + }, + "data_stream": { + "type": "metrics", + "dataset": "nginx.stubstatus", + "namespace": "ep" + }, + "agent": { + "type": "metricbeat", + "version": "7.11.0", + "hostname": "a73e7856c209", + "ephemeral_id": "1fbb4215-4ba3-42fa-9984-244b112c9a17", + "id": "2689a72c-6e18-45fe-b493-af1ec86af2b3", + "name": "a73e7856c209" } } \ No newline at end of file diff --git a/test/packages/nginx/docs/README.md b/test/packages/nginx/docs/README.md index b53627d3d8..53ebb724c7 100644 --- a/test/packages/nginx/docs/README.md +++ b/test/packages/nginx/docs/README.md @@ -1,21 +1,195 @@ # Nginx Integration +This integration periodically fetches metrics from [Nginx](https://nginx.org/) servers. It can parse access and error +logs created by the HTTP server. + +## Compatibility + +The Nginx `stubstatus` metrics was tested with Nginx 1.19.5 and are expected to work with all version >= 1.9. +The logs were tested with version 1.19.5. +On Windows, the module was tested with Nginx installed from the Chocolatey repository. + +## Logs + +**Timezone support** + +This datasource parses logs that don’t contain timezone information. For these logs, the Elastic Agent reads the local +timezone and uses it when parsing to convert the timestamp to UTC. The timezone to be used for parsing is included +in the event in the `event.timezone` field. + +To disable this conversion, the event.timezone field can be removed with the drop_fields processor. + +If logs are originated from systems or applications with a different timezone to the local one, the `event.timezone` +field can be overwritten with the original timezone using the add_fields processor. + +### Access Logs + +Access logs collects the nginx access logs. + +An example event for `access` looks as following: + +```json +{ + "agent": { + "hostname": "a73e7856c209", + "name": "a73e7856c209", + "id": "3987d2b3-b40a-4aa0-99fc-478f9d7079ea", + "ephemeral_id": "6d41da1c-5f71-4bd4-b326-a8913bfaa884", + "type": "filebeat", + "version": "7.11.0" + }, + "nginx": { + "access": { + "remote_ip_list": [ + "127.0.0.1" + ] + } + }, + "log": { + "file": { + "path": "/tmp/service_logs/access.log" + }, + "offset": 0 + }, + "elastic_agent": { + "id": "5ca3af72-37c3-48b6-92e8-176d154bb66f", + "version": "7.11.0", + "snapshot": true + }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "url": { + "original": "/server-status" + }, + "input": { + "type": "log" + }, + "@timestamp": "2020-12-03T11:41:57.000Z", + "ecs": { + "version": "1.6.0" + }, + "related": { + "ip": [ + "127.0.0.1" + ] + }, + "data_stream": { + "namespace": "ep", + "type": "logs", + "dataset": "nginx.access" + }, + "host": { + "hostname": "a73e7856c209", + "os": { + "kernel": "4.9.184-linuxkit", + "codename": "Core", + "name": "CentOS Linux", + "family": "redhat", + "version": "7 (Core)", + "platform": "centos" + }, + "containerized": true, + "ip": [ + "192.168.80.6" + ], + "name": "a73e7856c209", + "id": "06c26569966fd125c15acac5d7feffb6", + "mac": [ + "02:42:c0:a8:50:06" + ], + "architecture": "x86_64" + }, + "http": { + "request": { + "method": "get" + }, + "response": { + "status_code": 200, + "body": { + "bytes": 97 + } + }, + "version": "1.1" + }, + "event": { + "timezone": "+00:00", + "created": "2020-12-03T11:42:17.116Z", + "kind": "event", + "category": [ + "web" + ], + "type": [ + "access" + ], + "dataset": "nginx.access", + "outcome": "success" + }, + "user_agent": { + "original": "curl/7.64.0", + "name": "curl", + "device": { + "name": "Other" + }, + "version": "7.64.0" + } +} +``` + **Exported fields** | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | event.created contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from @timestamp in that @timestamp typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, @timestamp should be used. | date | -| http.request.method | HTTP request method. The value should retain its casing from the original event. For example, `GET`, `get`, and `GeT` are all considered valid values for this field. | keyword | +| destination.domain | Destination domain. | keyword | +| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | +| destination.port | Port of the destination. | long | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| event.created | Date/time when the event was first read by an agent, or by your pipeline. | date | +| event.dataset | Event dataset | constant_keyword | +| event.module | Event module | constant_keyword | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| http.request.method | HTTP request method. Prior to ECS 1.6.0 the following guidance was provided: "The field value must be normalized to lowercase for querying." As of ECS 1.6.0, the guidance is deprecated because the original case of the method may be useful in anomaly detection. Original case will be mandated in ECS 2.0.0 | keyword | | http.request.referrer | Referrer for this HTTP request. | keyword | | http.response.body.bytes | Size in bytes of the response body. | long | | http.response.status_code | HTTP response status code. | long | | http.version | HTTP version. | keyword | -| nginx.access.remote_ip_list | An array of remote IP addresses. It is a list because it is common to include, besides the client IP address, IP addresses from headers like `X-Forwarded-For`. Real source IP is restored to `source.ip`. | keyword | +| input.type | Input type | keyword | +| log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | +| log.offset | Log offset | long | +| nginx.access.remote_ip_list | An array of remote IP addresses. It is a list because it is common to include, besides the client IP address, IP addresses from headers like `X-Forwarded-For`. Real source IP is restored to `source.ip`. | array | | related.ip | All of the IPs seen on your event. | ip | | source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | | source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | @@ -24,11 +198,17 @@ | source.geo.continent_name | Name of the continent. | keyword | | source.geo.country_iso_code | Country ISO code. | keyword | | source.geo.country_name | Country name. | keyword | -| source.geo.location | | geo_point | +| source.geo.location | Longitude and latitude. | geo_point | | source.geo.region_iso_code | Region ISO code. | keyword | | source.geo.region_name | Region name. | keyword | | source.ip | IP address of the source (IPv4 or IPv6). | ip | -| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | keyword | +| tags | List of keywords used to tag each event. | keyword | +| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | +| url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | +| url.fragment | Portion of the url after the `#`, such as "top". The `#` is not part of the fragment. | keyword | +| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | +| url.path | Path of the request, such as "/search". | wildcard | +| url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | | user.name | Short name or login of the user. | keyword | | user_agent.device.name | Name of the device. | keyword | | user_agent.name | Name of the user agent. | keyword | @@ -39,31 +219,279 @@ | user_agent.version | Version of the user agent. | keyword | +### Error Logs + +Error logs collects the nginx error logs. + +An example event for `error` looks as following: + +```json +{ + "agent": { + "hostname": "a73e7856c209", + "name": "a73e7856c209", + "id": "3987d2b3-b40a-4aa0-99fc-478f9d7079ea", + "ephemeral_id": "6d41da1c-5f71-4bd4-b326-a8913bfaa884", + "type": "filebeat", + "version": "7.11.0" + }, + "process": { + "pid": 1, + "thread": { + "id": 1 + } + }, + "nginx": { + "error": {} + }, + "log": { + "file": { + "path": "/tmp/service_logs/error.log" + }, + "offset": 0, + "level": "warn" + }, + "elastic_agent": { + "id": "5ca3af72-37c3-48b6-92e8-176d154bb66f", + "version": "7.11.0", + "snapshot": true + }, + "message": "conflicting server name \"localhost\" on 0.0.0.0:80, ignored", + "input": { + "type": "log" + }, + "@timestamp": "2020-12-03T11:44:39.000Z", + "ecs": { + "version": "1.6.0" + }, + "data_stream": { + "namespace": "ep", + "type": "logs", + "dataset": "nginx.error" + }, + "host": { + "hostname": "a73e7856c209", + "os": { + "kernel": "4.9.184-linuxkit", + "codename": "Core", + "name": "CentOS Linux", + "family": "redhat", + "version": "7 (Core)", + "platform": "centos" + }, + "containerized": true, + "ip": [ + "192.168.80.6" + ], + "name": "a73e7856c209", + "id": "06c26569966fd125c15acac5d7feffb6", + "mac": [ + "02:42:c0:a8:50:06" + ], + "architecture": "x86_64" + }, + "event": { + "timezone": "+00:00", + "created": "2020-12-03T11:44:52.803Z", + "kind": "event", + "category": [ + "web" + ], + "type": [ + "error" + ], + "dataset": "nginx.error" + } +} +``` + **Exported fields** | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | event.created | Date/time when the event was first read by an agent, or by your pipeline. | date | +| event.dataset | Event dataset | constant_keyword | +| event.module | Event module | constant_keyword | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| input.type | Input type | keyword | +| log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | | log.level | Original log level of the log event. If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity). Some examples are `warn`, `err`, `i`, `informational`. | keyword | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | text | +| log.offset | Log offset | long | +| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | | nginx.error.connection_id | Connection identifier. | long | | process.pid | Process id. | long | | process.thread.id | Thread ID. | long | +| tags | List of keywords used to tag each event. | keyword | + + +## Metrics +### Stub Status Metrics + +The Nginx `stubstatus` stream collects data from the Nginx `ngx_http_stub_status` module. It scrapes the server status +data from the web page generated by `ngx_http_stub_status`. Please verify that your Nginx distribution comes with the mentioned +module and it's enabled in the Nginx configuration file: + +``` +location /nginx_status { + stub_status; + allow 127.0.0.1; # only allow requests from localhost + deny all; # deny all other hosts +} +``` + +It's highly recommended to replace `127.0.0.1` with your server’s IP address and make sure that this page accessible to only you. + +An example event for `stubstatus` looks as following: + +```json +{ + "@timestamp": "2020-12-03T11:47:31.996Z", + "host": { + "hostname": "a73e7856c209", + "architecture": "x86_64", + "os": { + "codename": "Core", + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.9.184-linuxkit" + }, + "name": "a73e7856c209", + "id": "06c26569966fd125c15acac5d7feffb6", + "containerized": true, + "ip": [ + "192.168.80.6" + ], + "mac": [ + "02:42:c0:a8:50:06" + ] + }, + "service": { + "type": "nginx", + "address": "http://elastic-package-service_nginx_1:80/server-status" + }, + "nginx": { + "stubstatus": { + "requests": 13, + "waiting": 0, + "hostname": "elastic-package-service_nginx_1:80", + "accepts": 13, + "handled": 13, + "current": 13, + "dropped": 0, + "writing": 1, + "active": 1, + "reading": 0 + } + }, + "elastic_agent": { + "snapshot": true, + "version": "7.11.0", + "id": "5ca3af72-37c3-48b6-92e8-176d154bb66f" + }, + "ecs": { + "version": "1.6.0" + }, + "event": { + "dataset": "nginx.stubstatus", + "module": "nginx", + "duration": 2231100 + }, + "metricset": { + "period": 10000, + "name": "stubstatus" + }, + "data_stream": { + "type": "metrics", + "dataset": "nginx.stubstatus", + "namespace": "ep" + }, + "agent": { + "type": "metricbeat", + "version": "7.11.0", + "hostname": "a73e7856c209", + "ephemeral_id": "1fbb4215-4ba3-42fa-9984-244b112c9a17", + "id": "2689a72c-6e18-45fe-b493-af1ec86af2b3", + "name": "a73e7856c209" + } +} +``` **Exported fields** | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | A user defined namespace. Namespaces are useful to allow grouping of data. Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`. Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions: \* Must not contain `-` \* No longer than 100 characters | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| event.dataset | Event dataset | constant_keyword | +| event.module | Event module | constant_keyword | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | nginx.stubstatus.accepts | The total number of accepted client connections. | long | | nginx.stubstatus.active | The current number of active client connections including Waiting connections. | long | | nginx.stubstatus.current | The current number of client requests. | long | @@ -74,5 +502,6 @@ | nginx.stubstatus.requests | The total number of client requests. | long | | nginx.stubstatus.waiting | The current number of idle client connections waiting for a request. | long | | nginx.stubstatus.writing | The current number of connections where Nginx is writing the response back to the client. | long | -| service.address | | keyword | +| service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | + diff --git a/test/packages/nginx/img/kibana-nginx.png b/test/packages/nginx/img/kibana-nginx.png deleted file mode 100644 index fa7b847b70..0000000000 Binary files a/test/packages/nginx/img/kibana-nginx.png and /dev/null differ diff --git a/test/packages/nginx/img/metricbeat-nginx.png b/test/packages/nginx/img/metricbeat-nginx.png deleted file mode 100644 index a1315ea194..0000000000 Binary files a/test/packages/nginx/img/metricbeat-nginx.png and /dev/null differ diff --git a/test/packages/nginx/img/nginx-logs-access-error.png b/test/packages/nginx/img/nginx-logs-access-error.png new file mode 100644 index 0000000000..fa54771eba Binary files /dev/null and b/test/packages/nginx/img/nginx-logs-access-error.png differ diff --git a/test/packages/nginx/img/nginx-logs-overview.png b/test/packages/nginx/img/nginx-logs-overview.png new file mode 100644 index 0000000000..9d657ee767 Binary files /dev/null and b/test/packages/nginx/img/nginx-logs-overview.png differ diff --git a/test/packages/nginx/img/nginx-metrics-overview.png b/test/packages/nginx/img/nginx-metrics-overview.png new file mode 100644 index 0000000000..442cac85b4 Binary files /dev/null and b/test/packages/nginx/img/nginx-metrics-overview.png differ diff --git a/test/packages/nginx/kibana/dashboard/nginx-023d2930-f1a5-11e7-a9ef-93c69af7b129.json b/test/packages/nginx/kibana/dashboard/nginx-023d2930-f1a5-11e7-a9ef-93c69af7b129.json index e65214bea0..a10d674849 100644 --- a/test/packages/nginx/kibana/dashboard/nginx-023d2930-f1a5-11e7-a9ef-93c69af7b129.json +++ b/test/packages/nginx/kibana/dashboard/nginx-023d2930-f1a5-11e7-a9ef-93c69af7b129.json @@ -20,69 +20,139 @@ }, "panelsJSON": [ { - "embeddableConfig": {}, + "embeddableConfig": { + "enhancements": {} + }, "gridData": { "h": 12, "i": "1", - "w": 24, - "x": 24, - "y": 0 + "w": 16, + "x": 0, + "y": 22 }, "panelIndex": "1", "panelRefName": "panel_0", - "version": "7.3.0" + "version": "7.9.3" }, { - "embeddableConfig": {}, + "embeddableConfig": { + "enhancements": {} + }, "gridData": { "h": 12, "i": "2", - "w": 24, - "x": 24, - "y": 12 + "w": 16, + "x": 16, + "y": 22 }, "panelIndex": "2", "panelRefName": "panel_1", - "version": "7.3.0" + "version": "7.9.3" }, { - "embeddableConfig": {}, + "embeddableConfig": { + "enhancements": {} + }, "gridData": { "h": 12, "i": "3", - "w": 24, - "x": 0, - "y": 12 + "w": 16, + "x": 32, + "y": 22 }, "panelIndex": "3", "panelRefName": "panel_2", - "version": "7.3.0" + "version": "7.9.3" }, { - "embeddableConfig": {}, + "embeddableConfig": { + "enhancements": {} + }, "gridData": { "h": 12, "i": "4", - "w": 24, + "w": 23, "x": 0, - "y": 0 + "y": 10 }, "panelIndex": "4", "panelRefName": "panel_3", - "version": "7.3.0" + "version": "7.9.3" }, { - "embeddableConfig": {}, + "embeddableConfig": { + "enhancements": {} + }, "gridData": { "h": 12, "i": "5", - "w": 48, - "x": 0, - "y": 24 + "w": 25, + "x": 23, + "y": 10 }, "panelIndex": "5", "panelRefName": "panel_4", - "version": "7.3.0" + "version": "7.9.3" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 5, + "i": "a6064ee6-47de-4c0f-81a0-69fbaca6e843", + "w": 9, + "x": 0, + "y": 0 + }, + "panelIndex": "a6064ee6-47de-4c0f-81a0-69fbaca6e843", + "panelRefName": "panel_5", + "version": "7.9.3" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 10, + "i": "98a96fa3-86b3-47b9-9b08-ed2bb0839702", + "w": 13, + "x": 9, + "y": 0 + }, + "panelIndex": "98a96fa3-86b3-47b9-9b08-ed2bb0839702", + "panelRefName": "panel_6", + "version": "7.9.3" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 10, + "i": "fec121fc-be02-4a94-8916-a0220a50ea8b", + "w": 13, + "x": 22, + "y": 0 + }, + "panelIndex": "fec121fc-be02-4a94-8916-a0220a50ea8b", + "panelRefName": "panel_7", + "version": "7.9.3" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 10, + "i": "0793bfb0-c51d-4684-82bd-4c7b64bfa1b7", + "w": 13, + "x": 35, + "y": 0 + }, + "panelIndex": "0793bfb0-c51d-4684-82bd-4c7b64bfa1b7", + "panelRefName": "panel_8", + "version": "7.9.3" } ], "timeRestore": false, @@ -91,7 +161,7 @@ }, "id": "nginx-023d2930-f1a5-11e7-a9ef-93c69af7b129", "migrationVersion": { - "dashboard": "7.3.0" + "dashboard": "7.11.0" }, "references": [ { @@ -118,6 +188,26 @@ "id": "nginx-dcbffe30-f1a4-11e7-a9ef-93c69af7b129", "name": "panel_4", "type": "visualization" + }, + { + "id": "nginx-ea7f9e10-3af6-11eb-94b7-0dab91df36a6", + "name": "panel_5", + "type": "visualization" + }, + { + "id": "nginx-7cc9ea40-3af8-11eb-94b7-0dab91df36a6", + "name": "panel_6", + "type": "visualization" + }, + { + "id": "nginx-823b3c80-3af9-11eb-94b7-0dab91df36a6", + "name": "panel_7", + "type": "visualization" + }, + { + "id": "nginx-e302b5a0-3afb-11eb-94b7-0dab91df36a6", + "name": "panel_8", + "type": "visualization" } ], "type": "dashboard" diff --git a/test/packages/nginx/kibana/dashboard/nginx-046212a0-a2a1-11e7-928f-5dbe6f6f5519.json b/test/packages/nginx/kibana/dashboard/nginx-046212a0-a2a1-11e7-928f-5dbe6f6f5519.json index 31a03a81af..3ca3b78492 100644 --- a/test/packages/nginx/kibana/dashboard/nginx-046212a0-a2a1-11e7-928f-5dbe6f6f5519.json +++ b/test/packages/nginx/kibana/dashboard/nginx-046212a0-a2a1-11e7-928f-5dbe6f6f5519.json @@ -14,7 +14,8 @@ } }, "optionsJSON": { - "darkTheme": false + "darkTheme": false, + "useMargins": true }, "panelsJSON": [ { @@ -23,6 +24,7 @@ "log.level", "message" ], + "enhancements": {}, "sort": [ "@timestamp", "desc" @@ -33,11 +35,11 @@ "i": "11", "w": 48, "x": 0, - "y": 16 + "y": 17 }, "panelIndex": "11", "panelRefName": "panel_0", - "version": "7.3.0" + "version": "7.9.3" }, { "embeddableConfig": { @@ -47,6 +49,7 @@ "http.response.status_code", "http.response.body.bytes" ], + "enhancements": {}, "sort": [ "@timestamp", "desc" @@ -57,37 +60,56 @@ "i": "16", "w": 48, "x": 0, - "y": 28 + "y": 29 }, "panelIndex": "16", "panelRefName": "panel_1", - "version": "7.3.0" + "version": "7.9.3" }, { - "embeddableConfig": {}, + "embeddableConfig": { + "enhancements": {} + }, "gridData": { "h": 12, "i": "18", "w": 48, "x": 0, - "y": 4 + "y": 5 }, "panelIndex": "18", "panelRefName": "panel_2", - "version": "7.3.0" + "version": "7.9.3" }, { - "embeddableConfig": {}, + "embeddableConfig": { + "enhancements": {} + }, "gridData": { - "h": 4, + "h": 5, "i": "19", - "w": 48, - "x": 0, + "w": 35, + "x": 13, "y": 0 }, "panelIndex": "19", "panelRefName": "panel_3", - "version": "7.3.0" + "version": "7.9.3" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 5, + "i": "723d15da-0fee-40a6-87f0-12c669f31b41", + "w": 13, + "x": 0, + "y": 0 + }, + "panelIndex": "723d15da-0fee-40a6-87f0-12c669f31b41", + "panelRefName": "panel_4", + "version": "7.9.3" } ], "timeRestore": false, @@ -96,7 +118,7 @@ }, "id": "nginx-046212a0-a2a1-11e7-928f-5dbe6f6f5519", "migrationVersion": { - "dashboard": "7.3.0" + "dashboard": "7.11.0" }, "references": [ { @@ -118,6 +140,11 @@ "id": "nginx-97109780-a2a5-11e7-928f-5dbe6f6f5519", "name": "panel_3", "type": "visualization" + }, + { + "id": "nginx-9484ecf0-3af5-11eb-94b7-0dab91df36a6", + "name": "panel_4", + "type": "visualization" } ], "type": "dashboard" diff --git a/test/packages/nginx/kibana/dashboard/nginx-55a9e6e0-a29e-11e7-928f-5dbe6f6f5519.json b/test/packages/nginx/kibana/dashboard/nginx-55a9e6e0-a29e-11e7-928f-5dbe6f6f5519.json index b4a5471eca..b6fa2301d7 100644 --- a/test/packages/nginx/kibana/dashboard/nginx-55a9e6e0-a29e-11e7-928f-5dbe6f6f5519.json +++ b/test/packages/nginx/kibana/dashboard/nginx-55a9e6e0-a29e-11e7-928f-5dbe6f6f5519.json @@ -14,24 +14,28 @@ } }, "optionsJSON": { - "darkTheme": false + "darkTheme": false, + "useMargins": true }, "panelsJSON": [ { - "embeddableConfig": {}, + "embeddableConfig": { + "enhancements": {} + }, "gridData": { "h": 12, "i": "3", "w": 12, "x": 36, - "y": 44 + "y": 45 }, "panelIndex": "3", "panelRefName": "panel_0", - "version": "7.3.0" + "version": "7.9.3" }, { "embeddableConfig": { + "enhancements": {}, "vis": { "legendOpen": true } @@ -41,14 +45,15 @@ "i": "4", "w": 12, "x": 24, - "y": 44 + "y": 45 }, "panelIndex": "4", "panelRefName": "panel_1", - "version": "7.3.0" + "version": "7.9.3" }, { "embeddableConfig": { + "enhancements": {}, "mapBounds": { "bottom_right": { "lat": -7.362466865535738, @@ -81,76 +86,101 @@ "i": "8", "w": 48, "x": 0, - "y": 4 + "y": 5 }, "panelIndex": "8", "panelRefName": "panel_2", - "version": "7.3.0" + "version": "7.9.3" }, { - "embeddableConfig": {}, + "embeddableConfig": { + "enhancements": {} + }, "gridData": { "h": 12, "i": "13", "w": 48, "x": 0, - "y": 20 + "y": 21 }, "panelIndex": "13", "panelRefName": "panel_3", - "version": "7.3.0" + "version": "7.9.3" }, { - "embeddableConfig": {}, + "embeddableConfig": { + "enhancements": {} + }, "gridData": { "h": 12, "i": "14", "w": 24, "x": 24, - "y": 32 + "y": 33 }, "panelIndex": "14", "panelRefName": "panel_4", - "version": "7.3.0" + "version": "7.9.3" }, { - "embeddableConfig": {}, + "embeddableConfig": { + "enhancements": {} + }, "gridData": { "h": 12, "i": "15", "w": 24, "x": 0, - "y": 32 + "y": 33 }, "panelIndex": "15", "panelRefName": "panel_5", - "version": "7.3.0" + "version": "7.9.3" }, { - "embeddableConfig": {}, + "embeddableConfig": { + "enhancements": {} + }, "gridData": { "h": 12, "i": "16", "w": 24, "x": 0, - "y": 44 + "y": 45 }, "panelIndex": "16", "panelRefName": "panel_6", - "version": "7.3.0" + "version": "7.9.3" }, { - "embeddableConfig": {}, + "embeddableConfig": { + "enhancements": {} + }, "gridData": { - "h": 4, + "h": 5, "i": "17", - "w": 48, - "x": 0, + "w": 35, + "x": 13, "y": 0 }, "panelIndex": "17", "panelRefName": "panel_7", - "version": "7.3.0" + "version": "7.9.3" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 5, + "i": "c66120eb-cc0f-49d6-be06-829653f24812", + "w": 13, + "x": 0, + "y": 0 + }, + "panelIndex": "c66120eb-cc0f-49d6-be06-829653f24812", + "panelRefName": "panel_8", + "version": "7.9.3" } ], "timeRestore": false, @@ -159,7 +189,7 @@ }, "id": "nginx-55a9e6e0-a29e-11e7-928f-5dbe6f6f5519", "migrationVersion": { - "dashboard": "7.3.0" + "dashboard": "7.11.0" }, "references": [ { @@ -201,6 +231,11 @@ "id": "nginx-97109780-a2a5-11e7-928f-5dbe6f6f5519", "name": "panel_7", "type": "visualization" + }, + { + "id": "nginx-9484ecf0-3af5-11eb-94b7-0dab91df36a6", + "name": "panel_8", + "type": "visualization" } ], "type": "dashboard" diff --git a/test/packages/nginx/kibana/ml_module/nginx-Logs-ml.json b/test/packages/nginx/kibana/ml_module/nginx-Logs-ml.json new file mode 100644 index 0000000000..53ad2045cd --- /dev/null +++ b/test/packages/nginx/kibana/ml_module/nginx-Logs-ml.json @@ -0,0 +1,419 @@ +{ + "attributes": { + "id": "nginx_data_stream", + "title": "Nginx access logs", + "description": "Find unusual activity in HTTP access logs.", + "type": "Web Access Logs", + "logo": { + "icon": "logoNginx" + }, + "defaultIndexPattern": "logs-*", + "query": { + "bool": { + "filter": [ + { + "term": { + "data_stream.dataset": "nginx.access" + } + }, + { + "exists": { + "field": "source.address" + } + }, + { + "exists": { + "field": "url.original" + } + }, + { + "exists": { + "field": "http.response.status_code" + } + } + ], + "must_not": { + "terms": { + "_tier": [ + "data_frozen", + "data_cold" + ] + } + } + } + }, + "jobs": [ + { + "id": "visitor_rate_nginx", + "config": { + "groups": [ + "nginx" + ], + "description": "HTTP Access Logs: Detect unusual visitor rates", + "analysis_config": { + "bucket_span": "15m", + "summary_count_field_name": "dc_source_address", + "detectors": [ + { + "detector_description": "Nginx access visitor rate", + "function": "non_zero_count" + } + ], + "influencers": [] + }, + "analysis_limits": { + "model_memory_limit": "10mb" + }, + "data_description": { + "time_field": "@timestamp", + "time_format": "epoch_ms" + }, + "model_plot_config": { + "enabled": true + }, + "custom_settings": { + "created_by": "ml-module-nginx-access-data-stream", + "custom_urls": [ + { + "url_name": "Nginx logs overview", + "url_value": "dashboards#/view/nginx-55a9e6e0-a29e-11e7-928f-5dbe6f6f5519?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))\u0026_a=(description:\u0027\u0027,filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:data_stream.dataset,negate:!f,params:(query:\u0027nginx.access\u0027),type:phrase,value:\u0027nginx.access\u0027),query:(match:(data_stream.dataset:(query:\u0027nginx.access\u0027,type:phrase))))),query:(language:kuery,query:\u0027\u0027))" + }, + { + "url_name": "Raw data", + "url_value": "discover#/?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))\u0026_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:data_stream.dataset,negate:!f,params:(query:\u0027nginx.access\u0027),type:phrase,value:\u0027nginx.access\u0027),query:(match:(data_stream.dataset:(query:\u0027nginx.access\u0027,type:phrase))))),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:\u0027\u0027),sort:!(\u0027@timestamp\u0027,desc))" + } + ] + } + } + }, + { + "id": "status_code_rate_nginx", + "config": { + "groups": [ + "nginx" + ], + "description": "HTTP Access Logs: Detect unusual status code rates", + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "Nginx access status code rate", + "function": "count", + "partition_field_name": "http.response.status_code" + } + ], + "influencers": [ + "http.response.status_code", + "source.address" + ] + }, + "analysis_limits": { + "model_memory_limit": "100mb" + }, + "data_description": { + "time_field": "@timestamp", + "time_format": "epoch_ms" + }, + "model_plot_config": { + "enabled": true + }, + "custom_settings": { + "created_by": "ml-module-nginx-access-data-stream", + "custom_urls": [ + { + "url_name": "Nginx logs overview", + "url_value": "dashboards#/view/nginx-55a9e6e0-a29e-11e7-928f-5dbe6f6f5519?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))\u0026_a=(description:\u0027\u0027,filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:data_stream.dataset,negate:!f,params:(query:\u0027nginx.access\u0027),type:phrase,value:\u0027nginx.access\u0027),query:(match:(data_stream.dataset:(query:\u0027nginx.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:http.response.status_code,negate:!f,params:(query:\u0027$http.response.status_code$\u0027),type:phrase,value:\u0027$http.response.status_code$\u0027),query:(match:(http.response.status_code:(query:\u0027$http.response.status_code$\u0027,type:phrase))))),query:(language:kuery,query:\u0027\u0027))" + }, + { + "url_name": "Raw data", + "url_value": "discover#/?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))\u0026_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:data_stream.dataset,negate:!f,params:(query:\u0027nginx.access\u0027),type:phrase,value:\u0027nginx.access\u0027),query:(match:(data_stream.dataset:(query:\u0027nginx.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:http.response.status_code,negate:!f,params:(query:\u0027$http.response.status_code$\u0027),type:phrase,value:\u0027$http.response.status_code$\u0027),query:(match:(http.response.status_code:(query:\u0027$http.response.status_code$\u0027,type:phrase))))),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:\u0027\u0027),sort:!(\u0027@timestamp\u0027,desc))" + } + ] + } + } + }, + { + "id": "source_ip_url_count_nginx", + "config": { + "groups": [ + "nginx" + ], + "description": "HTTP Access Logs: Detect unusual source IPs - high distinct count of URLs", + "analysis_config": { + "bucket_span": "1h", + "detectors": [ + { + "detector_description": "Nginx access source IP high dc URL", + "function": "high_distinct_count", + "field_name": "url.original", + "over_field_name": "source.address" + } + ], + "influencers": [ + "source.address" + ] + }, + "data_description": { + "time_field": "@timestamp", + "time_format": "epoch_ms" + }, + "custom_settings": { + "created_by": "ml-module-nginx-access-data-stream", + "custom_urls": [ + { + "url_name": "Nginx logs overview", + "url_value": "dashboards#/view/nginx-55a9e6e0-a29e-11e7-928f-5dbe6f6f5519?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))\u0026_a=(description:\u0027\u0027,filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:data_stream.dataset,negate:!f,params:(query:\u0027nginx.access\u0027),type:phrase,value:\u0027nginx.access\u0027),query:(match:(data_stream.dataset:(query:\u0027nginx.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:source.address,negate:!f,params:(query:\u0027$source.address$\u0027),type:phrase,value:\u0027$source.address$\u0027),query:(match:(source.address:(query:\u0027$source.address$\u0027,type:phrase))))),query:(language:kuery,query:\u0027\u0027))" + }, + { + "url_name": "Raw data", + "url_value": "discover#/?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))\u0026_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:data_stream.dataset,negate:!f,params:(query:\u0027nginx.access\u0027),type:phrase,value:\u0027nginx.access\u0027),query:(match:(data_stream.dataset:(query:\u0027nginx.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:source.address,negate:!f,params:(query:\u0027$source.address$\u0027),type:phrase,value:\u0027$source.address$\u0027),query:(match:(source.address:(query:\u0027$source.address$\u0027,type:phrase))))),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:\u0027\u0027),sort:!(\u0027@timestamp\u0027,desc))" + } + ] + } + } + }, + { + "id": "source_ip_request_rate_nginx", + "config": { + "groups": [ + "nginx" + ], + "description": "HTTP Access Logs: Detect unusual source IPs - high request rates", + "analysis_config": { + "bucket_span": "1h", + "detectors": [ + { + "detector_description": "Nginx access source IP high count", + "function": "high_count", + "over_field_name": "source.address" + } + ], + "influencers": [ + "source.address" + ] + }, + "data_description": { + "time_field": "@timestamp", + "time_format": "epoch_ms" + }, + "custom_settings": { + "created_by": "ml-module-nginx-access-data-stream", + "custom_urls": [ + { + "url_name": "Nginx logs overview", + "url_value": "dashboards#/view/nginx-55a9e6e0-a29e-11e7-928f-5dbe6f6f5519?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))\u0026_a=(description:\u0027\u0027,filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:data_stream.dataset,negate:!f,params:(query:\u0027nginx.access\u0027),type:phrase,value:\u0027nginx.access\u0027),query:(match:(data_stream.dataset:(query:\u0027nginx.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:source.address,negate:!f,params:(query:\u0027$source.address$\u0027),type:phrase,value:\u0027$source.address$\u0027),query:(match:(source.address:(query:\u0027$source.address$\u0027,type:phrase))))),query:(language:kuery,query:\u0027\u0027))" + }, + { + "url_name": "Raw data", + "url_value": "discover#/?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))\u0026_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:data_stream.dataset,negate:!f,params:(query:\u0027nginx.access\u0027),type:phrase,value:\u0027nginx.access\u0027),query:(match:(data_stream.dataset:(query:\u0027nginx.access\u0027,type:phrase)))),(\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:source.address,negate:!f,params:(query:\u0027$source.address$\u0027),type:phrase,value:\u0027$source.address$\u0027),query:(match:(source.address:(query:\u0027$source.address$\u0027,type:phrase))))),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:\u0027\u0027),sort:!(\u0027@timestamp\u0027,desc))" + } + ] + } + } + }, + { + "id": "low_request_rate_nginx", + "config": { + "groups": [ + "nginx" + ], + "description": "HTTP Access Logs: Detect low request rates", + "analysis_config": { + "bucket_span": "15m", + "summary_count_field_name": "doc_count", + "detectors": [ + { + "detector_description": "Nginx access low request rate", + "function": "low_count" + } + ], + "influencers": [] + }, + "analysis_limits": { + "model_memory_limit": "10mb" + }, + "data_description": { + "time_field": "@timestamp", + "time_format": "epoch_ms" + }, + "model_plot_config": { + "enabled": true + }, + "custom_settings": { + "created_by": "ml-module-nginx-access-data-stream", + "custom_urls": [ + { + "url_name": "Nginx logs overview", + "url_value": "dashboards#/view/nginx-55a9e6e0-a29e-11e7-928f-5dbe6f6f5519?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))\u0026_a=(description:\u0027\u0027,filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:data_stream.dataset,negate:!f,params:(query:\u0027nginx.access\u0027),type:phrase,value:\u0027nginx.access\u0027),query:(match:(data_stream.dataset:(query:\u0027nginx.access\u0027,type:phrase))))),query:(language:kuery,query:\u0027\u0027))" + }, + { + "url_name": "Raw data", + "url_value": "discover#/?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))\u0026_a=(columns:!(_source),filters:!((\u0027$state\u0027:(store:appState),meta:(alias:!n,disabled:!f,index:\u0027INDEX_PATTERN_ID\u0027,key:data_stream.dataset,negate:!f,params:(query:\u0027nginx.access\u0027),type:phrase,value:\u0027nginx.access\u0027),query:(match:(data_stream.dataset:(query:\u0027nginx.access\u0027,type:phrase))))),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:\u0027\u0027),sort:!(\u0027@timestamp\u0027,desc))" + } + ] + } + } + } + ], + "datafeeds": [ + { + "id": "datafeed-visitor_rate_nginx", + "job_id": "visitor_rate_nginx", + "config": { + "job_id": "visitor_rate_nginx", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "query": { + "bool": { + "filter": [ + { + "term": { + "data_stream.dataset": "nginx.access" + } + } + ] + } + }, + "aggregations": { + "buckets": { + "date_histogram": { + "field": "@timestamp", + "fixed_interval": "15m", + "offset": 0, + "order": { + "_key": "asc" + }, + "keyed": false, + "min_doc_count": 0 + }, + "aggregations": { + "@timestamp": { + "max": { + "field": "@timestamp" + } + }, + "dc_source_address": { + "cardinality": { + "field": "source.address" + } + } + } + } + } + } + }, + { + "id": "datafeed-status_code_rate_nginx", + "job_id": "status_code_rate_nginx", + "config": { + "job_id": "status_code_rate_nginx", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "query": { + "bool": { + "filter": [ + { + "term": { + "data_stream.dataset": "nginx.access" + } + } + ] + } + } + } + }, + { + "id": "datafeed-source_ip_url_count_nginx", + "job_id": "source_ip_url_count_nginx", + "config": { + "job_id": "source_ip_url_count_nginx", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "query": { + "bool": { + "filter": [ + { + "term": { + "data_stream.dataset": "nginx.access" + } + } + ] + } + } + } + }, + { + "id": "datafeed-source_ip_request_rate_nginx", + "job_id": "source_ip_request_rate_nginx", + "config": { + "job_id": "source_ip_request_rate_nginx", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "query": { + "bool": { + "filter": [ + { + "term": { + "data_stream.dataset": "nginx.access" + } + } + ] + } + } + } + }, + { + "id": "datafeed-low_request_rate_nginx", + "job_id": "low_request_rate_nginx", + "config": { + "job_id": "low_request_rate_nginx", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "query": { + "bool": { + "filter": [ + { + "term": { + "data_stream.dataset": "nginx.access" + } + } + ] + } + }, + "aggregations": { + "buckets": { + "date_histogram": { + "field": "@timestamp", + "fixed_interval": "15m", + "offset": 0, + "order": { + "_key": "asc" + }, + "keyed": false, + "min_doc_count": 0 + }, + "aggregations": { + "@timestamp": { + "max": { + "field": "@timestamp" + } + } + } + } + } + } + } + ] + }, + "id": "nginx-Logs-ml", + "migrationVersion": { + "search": "7.9.3" + }, + "references": [], + "type": "ml-module" +} \ No newline at end of file diff --git a/test/packages/nginx/kibana/search/nginx-6d9e66d0-a1f0-11e7-928f-5dbe6f6f5519.json b/test/packages/nginx/kibana/search/nginx-6d9e66d0-a1f0-11e7-928f-5dbe6f6f5519.json index 28877ec098..c7b7a54536 100644 --- a/test/packages/nginx/kibana/search/nginx-6d9e66d0-a1f0-11e7-928f-5dbe6f6f5519.json +++ b/test/packages/nginx/kibana/search/nginx-6d9e66d0-a1f0-11e7-928f-5dbe6f6f5519.json @@ -44,7 +44,7 @@ }, "id": "nginx-6d9e66d0-a1f0-11e7-928f-5dbe6f6f5519", "migrationVersion": { - "search": "7.4.0" + "search": "7.9.3" }, "references": [ { diff --git a/test/packages/nginx/kibana/search/nginx-9eb25600-a1f0-11e7-928f-5dbe6f6f5519.json b/test/packages/nginx/kibana/search/nginx-9eb25600-a1f0-11e7-928f-5dbe6f6f5519.json index 73fb4742ed..3e9d8af6f9 100644 --- a/test/packages/nginx/kibana/search/nginx-9eb25600-a1f0-11e7-928f-5dbe6f6f5519.json +++ b/test/packages/nginx/kibana/search/nginx-9eb25600-a1f0-11e7-928f-5dbe6f6f5519.json @@ -42,7 +42,7 @@ }, "id": "nginx-9eb25600-a1f0-11e7-928f-5dbe6f6f5519", "migrationVersion": { - "search": "7.4.0" + "search": "7.9.3" }, "references": [ { diff --git a/test/packages/nginx/kibana/search/nginx-Filebeat-Nginx-module.json b/test/packages/nginx/kibana/search/nginx-Filebeat-Nginx-module.json deleted file mode 100644 index 596158ae82..0000000000 --- a/test/packages/nginx/kibana/search/nginx-Filebeat-Nginx-module.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "attributes": { - "columns": [ - "url.original", - "http.request.method", - "http.response.status_code", - "http.request.referrer", - "http.response.body.bytes" - ], - "description": "", - "hits": 0, - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [], - "highlight": { - "fields": { - "*": {} - }, - "fragment_size": 2147483647, - "post_tags": [ - "@/kibana-highlighted-field@" - ], - "pre_tags": [ - "@kibana-highlighted-field@" - ], - "require_field_match": false - }, - "highlightAll": true, - "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", - "query": { - "language": "kuery", - "query": "(data_stream.dataset:nginx.access OR data_stream.dataset:nginx.error OR data_stream.dataset:nginx.ingress_controller)" - }, - "version": true - } - }, - "sort": [ - [ - "@timestamp", - "desc" - ] - ], - "title": "Nginx logs [Logs Nginx]", - "version": 1 - }, - "id": "nginx-Filebeat-Nginx-module", - "references": [ - { - "id": "logs-*", - "name": "kibanaSavedObjectMeta.searchSourceJSON.index", - "type": "index-pattern" - } - ], - "type": "search" -} \ No newline at end of file diff --git a/test/packages/nginx/kibana/search/nginx-Logs-Nginx-integration.json b/test/packages/nginx/kibana/search/nginx-Logs-Nginx-integration.json index fb048e2262..10cb44bee5 100644 --- a/test/packages/nginx/kibana/search/nginx-Logs-Nginx-integration.json +++ b/test/packages/nginx/kibana/search/nginx-Logs-Nginx-integration.json @@ -45,7 +45,7 @@ }, "id": "nginx-Logs-Nginx-integration", "migrationVersion": { - "search": "7.4.0" + "search": "7.9.3" }, "references": [ { diff --git a/test/packages/nginx/kibana/visualization/nginx-0dd6f320-a29f-11e7-928f-5dbe6f6f5519.json b/test/packages/nginx/kibana/visualization/nginx-0dd6f320-a29f-11e7-928f-5dbe6f6f5519.json index 357be477ac..23ff0811a3 100644 --- a/test/packages/nginx/kibana/visualization/nginx-0dd6f320-a29f-11e7-928f-5dbe6f6f5519.json +++ b/test/packages/nginx/kibana/visualization/nginx-0dd6f320-a29f-11e7-928f-5dbe6f6f5519.json @@ -2,7 +2,13 @@ "attributes": { "description": "", "kibanaSavedObjectMeta": { - "searchSourceJSON": {} + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } }, "title": "Data Volume [Logs Nginx]", "uiStateJSON": {}, @@ -12,6 +18,9 @@ "params": { "axis_formatter": "number", "axis_position": "left", + "axis_scale": "normal", + "default_index_pattern": "metrics-*", + "default_timefield": "@timestamp", "filter": { "language": "lucene", "query": "data_stream.dataset:nginx.access" @@ -19,16 +28,17 @@ "id": "61ca57f0-469d-11e7-af02-69e470af7417", "index_pattern": "logs-*", "interval": "auto", + "isModelInvalid": false, "legend_position": "bottom", "series": [ { "axis_position": "right", "chart_type": "line", "color": "#68BC00", - "fill": 0.5, + "fill": "0", "formatter": "bytes", "id": "61ca57f1-469d-11e7-af02-69e470af7417", - "label": "", + "label": "HTTP response body ", "line_width": 1, "metrics": [ { @@ -37,7 +47,7 @@ "type": "sum" } ], - "point_size": 1, + "point_size": "0", "seperate_axis": 0, "split_color_mode": "gradient", "split_filters": [ @@ -59,6 +69,7 @@ "show_grid": 1, "show_legend": 1, "time_field": "@timestamp", + "tooltip_mode": "show_all", "type": "timeseries" }, "title": "Data Volume [Logs Nginx]", @@ -67,7 +78,7 @@ }, "id": "nginx-0dd6f320-a29f-11e7-928f-5dbe6f6f5519", "migrationVersion": { - "visualization": "7.8.0" + "visualization": "7.9.3" }, "references": [], "type": "visualization" diff --git a/test/packages/nginx/kibana/visualization/nginx-1cfb1a80-a1f4-11e7-928f-5dbe6f6f5519.json b/test/packages/nginx/kibana/visualization/nginx-1cfb1a80-a1f4-11e7-928f-5dbe6f6f5519.json index b12b0dce44..8718798616 100644 --- a/test/packages/nginx/kibana/visualization/nginx-1cfb1a80-a1f4-11e7-928f-5dbe6f6f5519.json +++ b/test/packages/nginx/kibana/visualization/nginx-1cfb1a80-a1f4-11e7-928f-5dbe6f6f5519.json @@ -2,7 +2,13 @@ "attributes": { "description": "", "kibanaSavedObjectMeta": { - "searchSourceJSON": {} + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } }, "title": "Access logs over time [Logs Nginx]", "uiStateJSON": {}, @@ -23,11 +29,14 @@ ], "axis_formatter": "number", "axis_position": "left", + "axis_scale": "normal", "background_color_rules": [ { "id": "3189aa80-a1f3-11e7-a062-a1c3587f4874" } ], + "default_index_pattern": "metrics-*", + "default_timefield": "@timestamp", "filter": { "language": "lucene", "query": "data_stream.dataset:nginx.access" @@ -35,13 +44,14 @@ "id": "61ca57f0-469d-11e7-af02-69e470af7417", "index_pattern": "logs-*", "interval": "auto", + "isModelInvalid": false, "legend_position": "bottom", "series": [ { "axis_position": "right", "chart_type": "line", "color": "#68BC00", - "fill": 0.5, + "fill": "0", "formatter": "number", "id": "61ca57f1-469d-11e7-af02-69e470af7417", "label": "Access logs", @@ -52,7 +62,7 @@ "type": "count" } ], - "point_size": 1, + "point_size": "0", "seperate_axis": 0, "split_color_mode": "gradient", "split_filters": [ @@ -70,6 +80,7 @@ "show_grid": 1, "show_legend": 1, "time_field": "@timestamp", + "tooltip_mode": "show_all", "type": "timeseries" }, "title": "Access logs over time [Logs Nginx]", @@ -78,7 +89,7 @@ }, "id": "nginx-1cfb1a80-a1f4-11e7-928f-5dbe6f6f5519", "migrationVersion": { - "visualization": "7.8.0" + "visualization": "7.9.3" }, "references": [], "type": "visualization" diff --git a/test/packages/nginx/kibana/visualization/nginx-46322e50-a1f6-11e7-928f-5dbe6f6f5519.json b/test/packages/nginx/kibana/visualization/nginx-46322e50-a1f6-11e7-928f-5dbe6f6f5519.json index f6cb29ddba..bc597960ac 100644 --- a/test/packages/nginx/kibana/visualization/nginx-46322e50-a1f6-11e7-928f-5dbe6f6f5519.json +++ b/test/packages/nginx/kibana/visualization/nginx-46322e50-a1f6-11e7-928f-5dbe6f6f5519.json @@ -55,7 +55,7 @@ }, "id": "nginx-46322e50-a1f6-11e7-928f-5dbe6f6f5519", "migrationVersion": { - "visualization": "7.8.0" + "visualization": "7.9.3" }, "references": [], "type": "visualization" diff --git a/test/packages/nginx/kibana/visualization/nginx-47a8e0f0-f1a4-11e7-a9ef-93c69af7b129.json b/test/packages/nginx/kibana/visualization/nginx-47a8e0f0-f1a4-11e7-a9ef-93c69af7b129.json index 7f5c545941..eea9814391 100644 --- a/test/packages/nginx/kibana/visualization/nginx-47a8e0f0-f1a4-11e7-a9ef-93c69af7b129.json +++ b/test/packages/nginx/kibana/visualization/nginx-47a8e0f0-f1a4-11e7-a9ef-93c69af7b129.json @@ -2,7 +2,13 @@ "attributes": { "description": "", "kibanaSavedObjectMeta": { - "searchSourceJSON": {} + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } }, "title": "Active connections [Metrics Nginx]", "uiStateJSON": {}, @@ -12,19 +18,24 @@ "params": { "axis_formatter": "number", "axis_position": "left", + "axis_scale": "normal", + "default_index_pattern": "metrics-*", + "default_timefield": "@timestamp", "id": "61ca57f0-469d-11e7-af02-69e470af7417", "index_pattern": "metrics-*", "interval": "auto", + "isModelInvalid": false, "legend_position": "bottom", "series": [ { "axis_position": "right", "chart_type": "line", "color": "#68BC00", - "fill": 0.5, + "fill": "0", "formatter": "number", "id": "61ca57f1-469d-11e7-af02-69e470af7417", - "line_width": 1, + "label": "Active", + "line_width": "1", "metrics": [ { "field": "nginx.stubstatus.active", @@ -32,7 +43,7 @@ "type": "avg" } ], - "point_size": 1, + "point_size": "0", "seperate_axis": 0, "split_color_mode": "gradient", "split_mode": "everything", @@ -42,6 +53,7 @@ "show_grid": 1, "show_legend": 1, "time_field": "@timestamp", + "tooltip_mode": "show_all", "type": "timeseries" }, "title": "Active connections [Metrics Nginx]", @@ -50,7 +62,7 @@ }, "id": "nginx-47a8e0f0-f1a4-11e7-a9ef-93c69af7b129", "migrationVersion": { - "visualization": "7.8.0" + "visualization": "7.9.3" }, "references": [], "type": "visualization" diff --git a/test/packages/nginx/kibana/visualization/nginx-555df8a0-f1a1-11e7-a9ef-93c69af7b129.json b/test/packages/nginx/kibana/visualization/nginx-555df8a0-f1a1-11e7-a9ef-93c69af7b129.json index 72af9e6bb8..c0e4582c21 100644 --- a/test/packages/nginx/kibana/visualization/nginx-555df8a0-f1a1-11e7-a9ef-93c69af7b129.json +++ b/test/packages/nginx/kibana/visualization/nginx-555df8a0-f1a1-11e7-a9ef-93c69af7b129.json @@ -2,7 +2,13 @@ "attributes": { "description": "", "kibanaSavedObjectMeta": { - "searchSourceJSON": {} + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } }, "title": "Request Rate [Metrics Nginx]", "uiStateJSON": {}, @@ -12,16 +18,20 @@ "params": { "axis_formatter": "number", "axis_position": "left", + "axis_scale": "normal", + "default_index_pattern": "metrics-*", + "default_timefield": "@timestamp", "id": "61ca57f0-469d-11e7-af02-69e470af7417", "index_pattern": "metrics-*", "interval": "auto", + "isModelInvalid": false, "legend_position": "bottom", "series": [ { "axis_position": "right", "chart_type": "line", "color": "#68BC00", - "fill": 0.5, + "fill": "0", "formatter": "number", "id": "61ca57f1-469d-11e7-af02-69e470af7417", "label": "Request rate", @@ -39,7 +49,7 @@ "unit": "" } ], - "point_size": 1, + "point_size": "0", "seperate_axis": 0, "split_color_mode": "gradient", "split_mode": "everything", @@ -49,6 +59,7 @@ "show_grid": 1, "show_legend": 1, "time_field": "@timestamp", + "tooltip_mode": "show_all", "type": "timeseries" }, "title": "Request Rate [Metrics Nginx]", @@ -57,7 +68,7 @@ }, "id": "nginx-555df8a0-f1a1-11e7-a9ef-93c69af7b129", "migrationVersion": { - "visualization": "7.8.0" + "visualization": "7.9.3" }, "references": [], "type": "visualization" diff --git a/test/packages/nginx/kibana/visualization/nginx-7cc9ea40-3af8-11eb-94b7-0dab91df36a6.json b/test/packages/nginx/kibana/visualization/nginx-7cc9ea40-3af8-11eb-94b7-0dab91df36a6.json new file mode 100644 index 0000000000..d10f38378d --- /dev/null +++ b/test/packages/nginx/kibana/visualization/nginx-7cc9ea40-3af8-11eb-94b7-0dab91df36a6.json @@ -0,0 +1,141 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Total requests [Metrics Nginx]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "Total", + "field": "nginx.stubstatus.requests" + }, + "schema": "metric", + "type": "max" + }, + { + "enabled": true, + "id": "2", + "params": { + "drop_partials": false, + "extended_bounds": {}, + "field": "@timestamp", + "interval": "auto", + "min_doc_count": 1, + "scaleMetricValues": false, + "timeRange": { + "from": "now-15m", + "to": "now" + }, + "useNormalizedEsInterval": true + }, + "schema": "segment", + "type": "date_histogram" + } + ], + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "categoryAxes": [ + { + "id": "CategoryAxis-1", + "labels": { + "filter": true, + "show": true, + "truncate": 100 + }, + "position": "bottom", + "scale": { + "type": "linear" + }, + "show": true, + "style": {}, + "title": {}, + "type": "category" + } + ], + "grid": { + "categoryLines": false + }, + "labels": {}, + "legendPosition": "bottom", + "seriesParams": [ + { + "data": { + "id": "1", + "label": "Total" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + } + ], + "thresholdLine": { + "color": "#E7664C", + "show": false, + "style": "full", + "value": 10, + "width": 1 + }, + "times": [], + "type": "line", + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "LeftAxis-1", + "position": "left", + "scale": { + "mode": "normal", + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Total" + }, + "type": "value" + } + ] + }, + "title": "Total requests [Metrics Nginx]", + "type": "line" + } + }, + "id": "nginx-7cc9ea40-3af8-11eb-94b7-0dab91df36a6", + "migrationVersion": { + "visualization": "7.9.3" + }, + "references": [ + { + "id": "metrics-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/nginx/kibana/visualization/nginx-823b3c80-3af9-11eb-94b7-0dab91df36a6.json b/test/packages/nginx/kibana/visualization/nginx-823b3c80-3af9-11eb-94b7-0dab91df36a6.json new file mode 100644 index 0000000000..a9b46d54f8 --- /dev/null +++ b/test/packages/nginx/kibana/visualization/nginx-823b3c80-3af9-11eb-94b7-0dab91df36a6.json @@ -0,0 +1,141 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Processed requests [Metrics Nginx]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "Processed", + "field": "nginx.stubstatus.handled" + }, + "schema": "metric", + "type": "max" + }, + { + "enabled": true, + "id": "2", + "params": { + "drop_partials": false, + "extended_bounds": {}, + "field": "@timestamp", + "interval": "auto", + "min_doc_count": 1, + "scaleMetricValues": false, + "timeRange": { + "from": "now-15m", + "to": "now" + }, + "useNormalizedEsInterval": true + }, + "schema": "segment", + "type": "date_histogram" + } + ], + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "categoryAxes": [ + { + "id": "CategoryAxis-1", + "labels": { + "filter": true, + "show": true, + "truncate": 100 + }, + "position": "bottom", + "scale": { + "type": "linear" + }, + "show": true, + "style": {}, + "title": {}, + "type": "category" + } + ], + "grid": { + "categoryLines": false + }, + "labels": {}, + "legendPosition": "bottom", + "seriesParams": [ + { + "data": { + "id": "1", + "label": "Processed" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + } + ], + "thresholdLine": { + "color": "#E7664C", + "show": false, + "style": "full", + "value": 10, + "width": 1 + }, + "times": [], + "type": "line", + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "LeftAxis-1", + "position": "left", + "scale": { + "mode": "normal", + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Processed" + }, + "type": "value" + } + ] + }, + "title": "Processed requests [Metrics Nginx]", + "type": "line" + } + }, + "id": "nginx-823b3c80-3af9-11eb-94b7-0dab91df36a6", + "migrationVersion": { + "visualization": "7.9.3" + }, + "references": [ + { + "id": "metrics-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/nginx/kibana/visualization/nginx-9184fa00-a1f5-11e7-928f-5dbe6f6f5519.json b/test/packages/nginx/kibana/visualization/nginx-9184fa00-a1f5-11e7-928f-5dbe6f6f5519.json index 0f78a9f62b..291ffa7147 100644 --- a/test/packages/nginx/kibana/visualization/nginx-9184fa00-a1f5-11e7-928f-5dbe6f6f5519.json +++ b/test/packages/nginx/kibana/visualization/nginx-9184fa00-a1f5-11e7-928f-5dbe6f6f5519.json @@ -60,7 +60,7 @@ }, "id": "nginx-9184fa00-a1f5-11e7-928f-5dbe6f6f5519", "migrationVersion": { - "visualization": "7.8.0" + "visualization": "7.9.3" }, "references": [], "type": "visualization" diff --git a/test/packages/nginx/kibana/visualization/nginx-9484ecf0-3af5-11eb-94b7-0dab91df36a6.json b/test/packages/nginx/kibana/visualization/nginx-9484ecf0-3af5-11eb-94b7-0dab91df36a6.json new file mode 100644 index 0000000000..ed5a9ab56b --- /dev/null +++ b/test/packages/nginx/kibana/visualization/nginx-9484ecf0-3af5-11eb-94b7-0dab91df36a6.json @@ -0,0 +1,56 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Nginx instance [Logs Nginx]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "controls": [ + { + "fieldName": "host.hostname", + "id": "1607611131189", + "indexPatternRefName": "control_0_index_pattern", + "label": "Nginx instance", + "options": { + "dynamicOptions": true, + "multiselect": false, + "order": "desc", + "size": 5, + "type": "terms" + }, + "parent": "", + "type": "list" + } + ], + "pinFilters": false, + "updateFiltersOnChange": true, + "useTimeFilter": false + }, + "title": "Nginx instance [Logs Nginx]", + "type": "input_control_vis" + } + }, + "id": "nginx-9484ecf0-3af5-11eb-94b7-0dab91df36a6", + "migrationVersion": { + "visualization": "7.9.3" + }, + "references": [ + { + "id": "logs-*", + "name": "control_0_index_pattern", + "type": "index-pattern" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/nginx/kibana/visualization/nginx-97109780-a2a5-11e7-928f-5dbe6f6f5519.json b/test/packages/nginx/kibana/visualization/nginx-97109780-a2a5-11e7-928f-5dbe6f6f5519.json index 7b92656f74..e5ed94b6a2 100644 --- a/test/packages/nginx/kibana/visualization/nginx-97109780-a2a5-11e7-928f-5dbe6f6f5519.json +++ b/test/packages/nginx/kibana/visualization/nginx-97109780-a2a5-11e7-928f-5dbe6f6f5519.json @@ -19,7 +19,7 @@ }, "id": "nginx-97109780-a2a5-11e7-928f-5dbe6f6f5519", "migrationVersion": { - "visualization": "7.8.0" + "visualization": "7.9.3" }, "references": [], "type": "visualization" diff --git a/test/packages/nginx/kibana/visualization/nginx-Access-Browsers.json b/test/packages/nginx/kibana/visualization/nginx-Access-Browsers.json index 95fe836201..c5c27fcc69 100644 --- a/test/packages/nginx/kibana/visualization/nginx-Access-Browsers.json +++ b/test/packages/nginx/kibana/visualization/nginx-Access-Browsers.json @@ -62,7 +62,7 @@ }, "id": "nginx-Access-Browsers", "migrationVersion": { - "visualization": "7.8.0" + "visualization": "7.9.3" }, "references": [ { diff --git a/test/packages/nginx/kibana/visualization/nginx-Access-Map.json b/test/packages/nginx/kibana/visualization/nginx-Access-Map.json index 9f6a00fa83..45c0f9cfdd 100644 --- a/test/packages/nginx/kibana/visualization/nginx-Access-Map.json +++ b/test/packages/nginx/kibana/visualization/nginx-Access-Map.json @@ -70,7 +70,7 @@ }, "id": "nginx-Access-Map", "migrationVersion": { - "visualization": "7.8.0" + "visualization": "7.9.3" }, "references": [ { diff --git a/test/packages/nginx/kibana/visualization/nginx-Access-OSes.json b/test/packages/nginx/kibana/visualization/nginx-Access-OSes.json index 5a0b86673a..6227d9a144 100644 --- a/test/packages/nginx/kibana/visualization/nginx-Access-OSes.json +++ b/test/packages/nginx/kibana/visualization/nginx-Access-OSes.json @@ -28,8 +28,12 @@ "id": "2", "params": { "field": "user_agent.os.name", + "missingBucket": false, + "missingBucketLabel": "Missing", "order": "desc", "orderBy": "1", + "otherBucket": false, + "otherBucketLabel": "Other", "size": 5 }, "schema": "segment", @@ -40,29 +44,39 @@ "id": "3", "params": { "field": "user_agent.os.version", + "missingBucket": false, + "missingBucketLabel": "Missing", "order": "desc", "orderBy": "1", + "otherBucket": false, + "otherBucketLabel": "Other", "size": 5 }, "schema": "segment", "type": "terms" } ], - "listeners": {}, "params": { "addLegend": true, "addTooltip": true, "isDonut": true, + "labels": { + "last_level": true, + "show": false, + "truncate": 100, + "values": true + }, "legendPosition": "bottom", - "shareYAxis": true + "shareYAxis": true, + "type": "pie" }, - "title": "Nginx Access OSes", + "title": "Operating systems breakdown [Logs Nginx]", "type": "pie" } }, "id": "nginx-Access-OSes", "migrationVersion": { - "visualization": "7.8.0" + "visualization": "7.9.3" }, "references": [ { diff --git a/test/packages/nginx/kibana/visualization/nginx-a1d92240-f1a1-11e7-a9ef-93c69af7b129.json b/test/packages/nginx/kibana/visualization/nginx-a1d92240-f1a1-11e7-a9ef-93c69af7b129.json index c4a51c79c7..2bb31a5e48 100644 --- a/test/packages/nginx/kibana/visualization/nginx-a1d92240-f1a1-11e7-a9ef-93c69af7b129.json +++ b/test/packages/nginx/kibana/visualization/nginx-a1d92240-f1a1-11e7-a9ef-93c69af7b129.json @@ -2,7 +2,13 @@ "attributes": { "description": "", "kibanaSavedObjectMeta": { - "searchSourceJSON": {} + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } }, "title": "Accepts and Handled Rate [Metrics Nginx]", "uiStateJSON": {}, @@ -23,16 +29,20 @@ ], "axis_formatter": "number", "axis_position": "left", + "axis_scale": "normal", + "default_index_pattern": "metrics-*", + "default_timefield": "@timestamp", "id": "61ca57f0-469d-11e7-af02-69e470af7417", "index_pattern": "metrics-*", "interval": "auto", + "isModelInvalid": false, "legend_position": "bottom", "series": [ { "axis_position": "right", "chart_type": "line", "color": "#68BC00", - "fill": "0.5", + "fill": "", "formatter": "number", "id": "61ca57f1-469d-11e7-af02-69e470af7417", "label": "Accepts rate", @@ -50,7 +60,7 @@ "unit": "" } ], - "point_size": 1, + "point_size": "0", "seperate_axis": 0, "split_color_mode": "gradient", "split_mode": "everything", @@ -60,7 +70,7 @@ "axis_position": "right", "chart_type": "line", "color": "rgba(0,156,224,1)", - "fill": "0.9", + "fill": "0", "formatter": "number", "id": "56dd33b0-f1a3-11e7-95d0-8ddf041d42a2", "label": "Handled rate", @@ -78,7 +88,7 @@ "unit": "" } ], - "point_size": "3", + "point_size": "0", "seperate_axis": 0, "split_color_mode": "gradient", "split_mode": "everything", @@ -88,6 +98,7 @@ "show_grid": 1, "show_legend": 1, "time_field": "@timestamp", + "tooltip_mode": "show_all", "type": "timeseries" }, "title": "Accepts and Handled Rate [Metrics Nginx]", @@ -96,7 +107,7 @@ }, "id": "nginx-a1d92240-f1a1-11e7-a9ef-93c69af7b129", "migrationVersion": { - "visualization": "7.8.0" + "visualization": "7.9.3" }, "references": [], "type": "visualization" diff --git a/test/packages/nginx/kibana/visualization/nginx-b70b1b20-a1f4-11e7-928f-5dbe6f6f5519.json b/test/packages/nginx/kibana/visualization/nginx-b70b1b20-a1f4-11e7-928f-5dbe6f6f5519.json index 740fd6dfe1..3df0d8edf2 100644 --- a/test/packages/nginx/kibana/visualization/nginx-b70b1b20-a1f4-11e7-928f-5dbe6f6f5519.json +++ b/test/packages/nginx/kibana/visualization/nginx-b70b1b20-a1f4-11e7-928f-5dbe6f6f5519.json @@ -94,7 +94,7 @@ }, "id": "nginx-b70b1b20-a1f4-11e7-928f-5dbe6f6f5519", "migrationVersion": { - "visualization": "7.8.0" + "visualization": "7.9.3" }, "references": [], "type": "visualization" diff --git a/test/packages/nginx/kibana/visualization/nginx-d763a570-f1a1-11e7-a9ef-93c69af7b129.json b/test/packages/nginx/kibana/visualization/nginx-d763a570-f1a1-11e7-a9ef-93c69af7b129.json index 41964d37a8..0f8d58d94a 100644 --- a/test/packages/nginx/kibana/visualization/nginx-d763a570-f1a1-11e7-a9ef-93c69af7b129.json +++ b/test/packages/nginx/kibana/visualization/nginx-d763a570-f1a1-11e7-a9ef-93c69af7b129.json @@ -2,7 +2,13 @@ "attributes": { "description": "", "kibanaSavedObjectMeta": { - "searchSourceJSON": {} + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } }, "title": "Drops Rate [Metrics Nginx]", "uiStateJSON": {}, @@ -12,16 +18,20 @@ "params": { "axis_formatter": "number", "axis_position": "left", + "axis_scale": "normal", + "default_index_pattern": "metrics-*", + "default_timefield": "@timestamp", "id": "61ca57f0-469d-11e7-af02-69e470af7417", "index_pattern": "metrics-*", "interval": "auto", + "isModelInvalid": false, "legend_position": "bottom", "series": [ { "axis_position": "right", "chart_type": "line", "color": "rgba(188,0,65,1)", - "fill": 0.5, + "fill": "0", "formatter": "number", "id": "61ca57f1-469d-11e7-af02-69e470af7417", "label": "Drops rate", @@ -39,7 +49,7 @@ "unit": "" } ], - "point_size": 1, + "point_size": "0", "seperate_axis": 0, "split_color_mode": "gradient", "split_mode": "everything", @@ -49,6 +59,7 @@ "show_grid": 1, "show_legend": 1, "time_field": "@timestamp", + "tooltip_mode": "show_all", "type": "timeseries" }, "title": "Drops Rate [Metrics Nginx]", @@ -57,7 +68,7 @@ }, "id": "nginx-d763a570-f1a1-11e7-a9ef-93c69af7b129", "migrationVersion": { - "visualization": "7.8.0" + "visualization": "7.9.3" }, "references": [], "type": "visualization" diff --git a/test/packages/nginx/kibana/visualization/nginx-dcbffe30-f1a4-11e7-a9ef-93c69af7b129.json b/test/packages/nginx/kibana/visualization/nginx-dcbffe30-f1a4-11e7-a9ef-93c69af7b129.json index dc7e48be35..948b78d829 100644 --- a/test/packages/nginx/kibana/visualization/nginx-dcbffe30-f1a4-11e7-a9ef-93c69af7b129.json +++ b/test/packages/nginx/kibana/visualization/nginx-dcbffe30-f1a4-11e7-a9ef-93c69af7b129.json @@ -2,7 +2,13 @@ "attributes": { "description": "", "kibanaSavedObjectMeta": { - "searchSourceJSON": {} + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } }, "title": "Reading / Writing / Waiting Rates [Metrics Nginx]", "uiStateJSON": {}, @@ -12,16 +18,20 @@ "params": { "axis_formatter": "number", "axis_position": "left", + "axis_scale": "normal", + "default_index_pattern": "metrics-*", + "default_timefield": "@timestamp", "id": "61ca57f0-469d-11e7-af02-69e470af7417", "index_pattern": "metrics-*", "interval": "auto", + "isModelInvalid": false, "legend_position": "bottom", "series": [ { "axis_position": "right", "chart_type": "line", "color": "#68BC00", - "fill": 0.5, + "fill": "0", "formatter": "number", "id": "61ca57f1-469d-11e7-af02-69e470af7417", "label": "Reading", @@ -33,7 +43,7 @@ "type": "avg" } ], - "point_size": 1, + "point_size": "0", "seperate_axis": 0, "split_color_mode": "gradient", "split_mode": "everything", @@ -43,7 +53,7 @@ "axis_position": "right", "chart_type": "line", "color": "rgba(0,156,224,1)", - "fill": 0.5, + "fill": "0", "formatter": "number", "id": "b1773680-f1a4-11e7-95d0-8ddf041d42a2", "label": "Writing", @@ -55,7 +65,7 @@ "type": "avg" } ], - "point_size": 1, + "point_size": "0", "seperate_axis": 0, "split_color_mode": "gradient", "split_mode": "everything", @@ -65,7 +75,7 @@ "axis_position": "right", "chart_type": "line", "color": "rgba(252,220,0,1)", - "fill": 0.5, + "fill": "0", "formatter": "number", "id": "b68aa6c0-f1a4-11e7-95d0-8ddf041d42a2", "label": "Waiting", @@ -77,7 +87,7 @@ "type": "avg" } ], - "point_size": 1, + "point_size": "0", "seperate_axis": 0, "split_color_mode": "gradient", "split_mode": "everything", @@ -87,6 +97,7 @@ "show_grid": 1, "show_legend": 1, "time_field": "@timestamp", + "tooltip_mode": "show_all", "type": "timeseries" }, "title": "Reading / Writing / Waiting Rates [Metrics Nginx]", @@ -95,7 +106,7 @@ }, "id": "nginx-dcbffe30-f1a4-11e7-a9ef-93c69af7b129", "migrationVersion": { - "visualization": "7.8.0" + "visualization": "7.9.3" }, "references": [], "type": "visualization" diff --git a/test/packages/nginx/kibana/visualization/nginx-e302b5a0-3afb-11eb-94b7-0dab91df36a6.json b/test/packages/nginx/kibana/visualization/nginx-e302b5a0-3afb-11eb-94b7-0dab91df36a6.json new file mode 100644 index 0000000000..346e617d11 --- /dev/null +++ b/test/packages/nginx/kibana/visualization/nginx-e302b5a0-3afb-11eb-94b7-0dab91df36a6.json @@ -0,0 +1,141 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Heartbeat / Up [Metrics Nginx]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "Up", + "field": "host.hostname" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "2", + "params": { + "drop_partials": false, + "extended_bounds": {}, + "field": "@timestamp", + "interval": "auto", + "min_doc_count": 1, + "scaleMetricValues": false, + "timeRange": { + "from": "now-15m", + "to": "now" + }, + "useNormalizedEsInterval": true + }, + "schema": "segment", + "type": "date_histogram" + } + ], + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "categoryAxes": [ + { + "id": "CategoryAxis-1", + "labels": { + "filter": true, + "show": true, + "truncate": 100 + }, + "position": "bottom", + "scale": { + "type": "linear" + }, + "show": true, + "style": {}, + "title": {}, + "type": "category" + } + ], + "grid": { + "categoryLines": false + }, + "labels": {}, + "legendPosition": "bottom", + "seriesParams": [ + { + "data": { + "id": "1", + "label": "Up" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "lineWidth": 2, + "mode": "normal", + "show": true, + "showCircles": false, + "type": "line", + "valueAxis": "ValueAxis-1" + } + ], + "thresholdLine": { + "color": "#E7664C", + "show": false, + "style": "full", + "value": 10, + "width": 1 + }, + "times": [], + "type": "line", + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "LeftAxis-1", + "position": "left", + "scale": { + "mode": "normal", + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Up" + }, + "type": "value" + } + ] + }, + "title": "Heartbeat / Up [Metrics Nginx]", + "type": "line" + } + }, + "id": "nginx-e302b5a0-3afb-11eb-94b7-0dab91df36a6", + "migrationVersion": { + "visualization": "7.9.3" + }, + "references": [ + { + "id": "metrics-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/nginx/kibana/visualization/nginx-ea7f9e10-3af6-11eb-94b7-0dab91df36a6.json b/test/packages/nginx/kibana/visualization/nginx-ea7f9e10-3af6-11eb-94b7-0dab91df36a6.json new file mode 100644 index 0000000000..6de0f9bf9e --- /dev/null +++ b/test/packages/nginx/kibana/visualization/nginx-ea7f9e10-3af6-11eb-94b7-0dab91df36a6.json @@ -0,0 +1,56 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Nginx instance [Metrics Nginx]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "controls": [ + { + "fieldName": "host.hostname", + "id": "1607611131189", + "indexPatternRefName": "control_0_index_pattern", + "label": "Nginx instance", + "options": { + "dynamicOptions": true, + "multiselect": false, + "order": "desc", + "size": 5, + "type": "terms" + }, + "parent": "", + "type": "list" + } + ], + "pinFilters": false, + "updateFiltersOnChange": true, + "useTimeFilter": false + }, + "title": "Nginx instance [Metrics Nginx]", + "type": "input_control_vis" + } + }, + "id": "nginx-ea7f9e10-3af6-11eb-94b7-0dab91df36a6", + "migrationVersion": { + "visualization": "7.9.3" + }, + "references": [ + { + "id": "metrics-*", + "name": "control_0_index_pattern", + "type": "index-pattern" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/test/packages/nginx/manifest.yml b/test/packages/nginx/manifest.yml index f2ab43c6f6..717b6a5973 100644 --- a/test/packages/nginx/manifest.yml +++ b/test/packages/nginx/manifest.yml @@ -1,27 +1,28 @@ format_version: 1.0.0 name: nginx title: Nginx -# version is set to something very large to so this test package can -# be installed in the package registry regardless of the version of -# the actual apache package in the registry at any given time. version: 999.999.999 license: basic -description: Nginx Integration +description: Collect logs and metrics from Nginx HTTP servers with Elastic Agent. type: integration categories: - web - security -release: experimental +release: ga conditions: - kibana.version: '^7.9.0' + kibana.version: "^7.14.0 || ^8.0.0" screenshots: - - src: /img/kibana-nginx.png - title: kibana nginx - size: 1218x1266 + - src: /img/nginx-metrics-overview.png + title: Nginx metrics overview + size: 3360x2302 type: image/png - - src: /img/metricbeat-nginx.png - title: metricbeat nginx - size: 2560x2100 + - src: /img/nginx-logs-access-error.png + title: Nginx access and error logs + size: 3360x3590 + type: image/png + - src: /img/nginx-logs-overview.png + title: Nginx logs overview + size: 3360x3590 type: image/png icons: - src: /img/logo_nginx.svg @@ -35,7 +36,66 @@ policy_templates: inputs: - type: logfile title: Collect logs from Nginx instances - description: Collecting Nginx access, error and ingress controller logs + description: Collecting Nginx access and error logs + - type: httpjson + title: Collect logs from third-party REST API (experimental) + description: Collect logs from third-party REST API (experimental) + vars: + - name: url + type: text + title: URL of Splunk Enterprise Server + description: i.e. scheme://host:port, path is automatic + show_user: true + required: true + default: https://server.example.com:8089 + - name: username + type: text + title: Splunk REST API Username + show_user: true + required: false + - name: password + type: password + title: Splunk REST API Password + show_user: true + required: false + - name: token + type: password + title: Splunk Authorization Token + description: | + Bearer Token or Session Key, e.g. "Bearer eyJFd3e46..." + or "Splunk 192fd3e...". Cannot be used with username + and password. + show_user: true + required: false + - name: ssl + type: yaml + title: SSL Configuration + description: i.e. certificate_authorities, supported_protocols, verification_mode etc. + multi: false + required: false + show_user: false + default: | + #certificate_authorities: + # - | + # -----BEGIN CERTIFICATE----- + # MIIDCjCCAfKgAwIBAgITJ706Mu2wJlKckpIvkWxEHvEyijANBgkqhkiG9w0BAQsF + # ADAUMRIwEAYDVQQDDAlsb2NhbGhvc3QwIBcNMTkwNzIyMTkyOTA0WhgPMjExOTA2 + # MjgxOTI5MDRaMBQxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEB + # BQADggEPADCCAQoCggEBANce58Y/JykI58iyOXpxGfw0/gMvF0hUQAcUrSMxEO6n + # fZRA49b4OV4SwWmA3395uL2eB2NB8y8qdQ9muXUdPBWE4l9rMZ6gmfu90N5B5uEl + # 94NcfBfYOKi1fJQ9i7WKhTjlRkMCgBkWPkUokvBZFRt8RtF7zI77BSEorHGQCk9t + # /D7BS0GJyfVEhftbWcFEAG3VRcoMhF7kUzYwp+qESoriFRYLeDWv68ZOvG7eoWnP + # PsvZStEVEimjvK5NSESEQa9xWyJOmlOKXhkdymtcUd/nXnx6UTCFgnkgzSdTWV41 + # CI6B6aJ9svCTI2QuoIq2HxX/ix7OvW1huVmcyHVxyUECAwEAAaNTMFEwHQYDVR0O + # BBYEFPwN1OceFGm9v6ux8G+DZ3TUDYxqMB8GA1UdIwQYMBaAFPwN1OceFGm9v6ux + # 8G+DZ3TUDYxqMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAG5D + # 874A4YI7YUwOVsVAdbWtgp1d0zKcPRR+r2OdSbTAV5/gcS3jgBJ3i1BN34JuDVFw + # 3DeJSYT3nxy2Y56lLnxDeF8CUTUtVQx3CuGkRg1ouGAHpO/6OqOhwLLorEmxi7tA + # H2O8mtT0poX5AnOAhzVy7QW0D/k4WaoLyckM5hUa6RtvgvLxOwA0U+VGurCDoctu + # 8F4QOgTAWyh8EZIwaKCliFRSynDpv3JTUwtfZkxo6K6nce1RhCWFAsMvDZL8Dgc0 + # yvgJ38BRsFOtkRuAGSf6ZUwTO8JJRRIFnpUzXflAnGivK9M13D5GEQMmIl6U9Pvk + # sxSmbIUfc2SGJGCJD4I= + # -----END CERTIFICATE----- - type: nginx/metrics vars: - name: hosts