From 0bc365633564aa964ff96cfcdf15ff9a37378602 Mon Sep 17 00:00:00 2001 From: Sai Kiran <85323324+r00tu53r@users.noreply.github.com> Date: Wed, 12 Jan 2022 14:21:39 +1100 Subject: [PATCH 1/3] Upgrade ECS to 8.0.0 --- packages/nginx/_dev/build/build.yml | 2 +- packages/nginx/changelog.yml | 5 + .../pipeline/test-access.log-expected.json | 72 ++++----- .../pipeline/test-nginx.log-expected.json | 28 ++-- .../test-test-with-host.log-expected.json | 22 +-- .../elasticsearch/ingest_pipeline/default.yml | 5 +- .../data_stream/access/sample_event.json | 151 +++++++++--------- .../pipeline/test-error-raw.log-expected.json | 16 +- .../elasticsearch/ingest_pipeline/default.yml | 2 +- .../nginx/data_stream/error/sample_event.json | 125 ++++++++------- .../data_stream/stubstatus/sample_event.json | 114 ++++++------- packages/nginx/docs/README.md | 4 +- packages/nginx/manifest.yml | 2 +- 13 files changed, 282 insertions(+), 266 deletions(-) diff --git a/packages/nginx/_dev/build/build.yml b/packages/nginx/_dev/build/build.yml index 08d85edcf9a..809e76063e9 100644 --- a/packages/nginx/_dev/build/build.yml +++ b/packages/nginx/_dev/build/build.yml @@ -1,3 +1,3 @@ dependencies: ecs: - reference: git@1.12 + reference: git@8.0 diff --git a/packages/nginx/changelog.yml b/packages/nginx/changelog.yml index 44ec8316cf9..e0754578df7 100644 --- a/packages/nginx/changelog.yml +++ b/packages/nginx/changelog.yml @@ -1,3 +1,8 @@ +- version: "1.3.0" + changes: + - description: Update to ECS 8.0 + type: enhancement + link: https://github.com/elastic/integrations/pull/2505 # newer versions go on top - version: "1.2.3" changes: diff --git a/packages/nginx/data_stream/access/_dev/test/pipeline/test-access.log-expected.json b/packages/nginx/data_stream/access/_dev/test/pipeline/test-access.log-expected.json index 6013a669fdc..7c236cd87b4 100644 --- a/packages/nginx/data_stream/access/_dev/test/pipeline/test-access.log-expected.json +++ b/packages/nginx/data_stream/access/_dev/test/pipeline/test-access.log-expected.json @@ -33,7 +33,7 @@ ], "@timestamp": "2016-10-25T12:49:33.000Z", "ecs": { - "version": "1.12.0" + "version": "8.0.0" }, "_tmp": {}, "related": { @@ -43,7 +43,7 @@ }, "http": { "request": { - "method": "get" + "method": "GET" }, "version": "1.1", "response": { @@ -54,7 +54,7 @@ } }, "event": { - "ingested": "2021-12-14T14:48:32.282353575Z", + "ingested": "2022-01-12T03:18:12.673228241Z", "original": "67.43.156.13 - - [25/Oct/2016:14:49:33 +0200] \"GET / HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36\"", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -114,7 +114,7 @@ ], "@timestamp": "2016-10-25T12:49:34.000Z", "ecs": { - "version": "1.12.0" + "version": "8.0.0" }, "_tmp": {}, "related": { @@ -124,7 +124,7 @@ }, "http": { "request": { - "method": "get", + "method": "GET", "referrer": "http://localhost:8080/" }, "version": "1.1", @@ -136,7 +136,7 @@ } }, "event": { - "ingested": "2021-12-14T14:48:32.282356218Z", + "ingested": "2022-01-12T03:18:12.673230472Z", "original": "67.43.156.13 - - [25/Oct/2016:14:49:34 +0200] \"GET /favicon.ico HTTP/1.1\" 404 571 \"http://localhost:8080/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36\"", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -195,7 +195,7 @@ ], "@timestamp": "2016-10-25T12:50:44.000Z", "ecs": { - "version": "1.12.0" + "version": "8.0.0" }, "_tmp": {}, "related": { @@ -205,7 +205,7 @@ }, "http": { "request": { - "method": "get" + "method": "GET" }, "version": "1.1", "response": { @@ -216,7 +216,7 @@ } }, "event": { - "ingested": "2021-12-14T14:48:32.282356644Z", + "ingested": "2022-01-12T03:18:12.673231678Z", "original": "67.43.156.13 - - [25/Oct/2016:14:50:44 +0200] \"GET /adsasd HTTP/1.1\" 404 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36\"", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -275,7 +275,7 @@ ], "@timestamp": "2016-12-07T09:34:43.000Z", "ecs": { - "version": "1.12.0" + "version": "8.0.0" }, "_tmp": {}, "related": { @@ -285,7 +285,7 @@ }, "http": { "request": { - "method": "get" + "method": "GET" }, "version": "1.1", "response": { @@ -296,7 +296,7 @@ } }, "event": { - "ingested": "2021-12-14T14:48:32.282357030Z", + "ingested": "2022-01-12T03:18:12.673232740Z", "original": "67.43.156.13 - - [07/Dec/2016:10:34:43 +0100] \"GET / HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -356,7 +356,7 @@ ], "@timestamp": "2016-12-07T09:34:43.000Z", "ecs": { - "version": "1.12.0" + "version": "8.0.0" }, "_tmp": {}, "related": { @@ -366,7 +366,7 @@ }, "http": { "request": { - "method": "get", + "method": "GET", "referrer": "http://localhost:8080/" }, "version": "1.1", @@ -378,7 +378,7 @@ } }, "event": { - "ingested": "2021-12-14T14:48:32.282357376Z", + "ingested": "2022-01-12T03:18:12.673233809Z", "original": "67.43.156.13 - - [07/Dec/2016:10:34:43 +0100] \"GET /favicon.ico HTTP/1.1\" 404 571 \"http://localhost:8080/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -437,7 +437,7 @@ ], "@timestamp": "2016-12-07T09:43:18.000Z", "ecs": { - "version": "1.12.0" + "version": "8.0.0" }, "_tmp": {}, "related": { @@ -447,7 +447,7 @@ }, "http": { "request": { - "method": "get" + "method": "GET" }, "version": "1.1", "response": { @@ -458,7 +458,7 @@ } }, "event": { - "ingested": "2021-12-14T14:48:32.282357746Z", + "ingested": "2022-01-12T03:18:12.673234868Z", "original": "67.43.156.13 - - [07/Dec/2016:10:43:18 +0100] \"GET /test HTTP/1.1\" 404 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -517,7 +517,7 @@ ], "@timestamp": "2016-12-07T09:43:21.000Z", "ecs": { - "version": "1.12.0" + "version": "8.0.0" }, "_tmp": {}, "related": { @@ -527,7 +527,7 @@ }, "http": { "request": { - "method": "get" + "method": "GET" }, "version": "1.1", "response": { @@ -538,7 +538,7 @@ } }, "event": { - "ingested": "2021-12-14T14:48:32.282358112Z", + "ingested": "2022-01-12T03:18:12.673235892Z", "original": "67.43.156.13 - - [07/Dec/2016:10:43:21 +0100] \"GET /test HTTP/1.1\" 404 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -597,7 +597,7 @@ ], "@timestamp": "2016-12-07T09:43:23.000Z", "ecs": { - "version": "1.12.0" + "version": "8.0.0" }, "_tmp": {}, "related": { @@ -607,7 +607,7 @@ }, "http": { "request": { - "method": "get" + "method": "GET" }, "version": "1.1", "response": { @@ -618,7 +618,7 @@ } }, "event": { - "ingested": "2021-12-14T14:48:32.282358462Z", + "ingested": "2022-01-12T03:18:12.673236930Z", "original": "67.43.156.13 - - [07/Dec/2016:10:43:23 +0100] \"GET /test1 HTTP/1.1\" 404 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -665,7 +665,7 @@ ], "@timestamp": "2016-12-07T10:04:37.000Z", "ecs": { - "version": "1.12.0" + "version": "8.0.0" }, "_tmp": {}, "related": { @@ -675,7 +675,7 @@ }, "http": { "request": { - "method": "get" + "method": "GET" }, "version": "1.1", "response": { @@ -686,7 +686,7 @@ } }, "event": { - "ingested": "2021-12-14T14:48:32.282358812Z", + "ingested": "2022-01-12T03:18:12.673238006Z", "original": "127.0.0.1 - - [07/Dec/2016:11:04:37 +0100] \"GET /test1 HTTP/1.1\" 404 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -733,7 +733,7 @@ ], "@timestamp": "2016-12-07T10:04:58.000Z", "ecs": { - "version": "1.12.0" + "version": "8.0.0" }, "_tmp": {}, "related": { @@ -743,7 +743,7 @@ }, "http": { "request": { - "method": "get" + "method": "GET" }, "version": "1.1", "response": { @@ -754,7 +754,7 @@ } }, "event": { - "ingested": "2021-12-14T14:48:32.282359158Z", + "ingested": "2022-01-12T03:18:12.673239059Z", "original": "127.0.0.1 - - [07/Dec/2016:11:04:58 +0100] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -801,7 +801,7 @@ ], "@timestamp": "2016-12-07T10:04:59.000Z", "ecs": { - "version": "1.12.0" + "version": "8.0.0" }, "_tmp": {}, "related": { @@ -811,7 +811,7 @@ }, "http": { "request": { - "method": "get" + "method": "GET" }, "version": "1.1", "response": { @@ -822,7 +822,7 @@ } }, "event": { - "ingested": "2021-12-14T14:48:32.282359505Z", + "ingested": "2022-01-12T03:18:12.673240124Z", "original": "127.0.0.1 - - [07/Dec/2016:11:04:59 +0100] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -869,7 +869,7 @@ ], "@timestamp": "2016-12-07T10:05:07.000Z", "ecs": { - "version": "1.12.0" + "version": "8.0.0" }, "_tmp": {}, "related": { @@ -879,7 +879,7 @@ }, "http": { "request": { - "method": "get" + "method": "GET" }, "version": "1.1", "response": { @@ -890,7 +890,7 @@ } }, "event": { - "ingested": "2021-12-14T14:48:32.282360050Z", + "ingested": "2022-01-12T03:18:12.673241307Z", "original": "127.0.0.1 - - [07/Dec/2016:11:05:07 +0100] \"GET /taga HTTP/1.1\" 404 169 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"\nlessons.example.com 192.168.0.1 - - [09/Jun/2020:12:10:39 -0700] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 206 7648063 \"http://lessons.example.com/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4\" \"Mozilla/5.0 (Linux; Android 5.1.1; KFFOWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/81.2.16 like Chrome/81.0.4044.138 Safari/537.36\"\nlessons.example.com 192.168.0.1 - - [09/Jun/2020:12:15:39 -0700] \"GET /%D0%A0%D1%83%D1%81%D1%81%D0%BA%D0%B0%D1%8F%20%D1%88%D0%BA%D0%BE%D0%BB%D0%B0%20-%20InternetUrok%201%D0%BA%D0%BB%D0%B0%D1%81%D1%81/ HTTP/1.1\" 206 7648063 \"http://lessons.example.com/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4\" \"Mozilla/5.0 (Linux; Android 5.1.1; KFFOWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/81.2.16 like Chrome/81.0.4044.138 Safari/537.36\"", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/nginx/data_stream/access/_dev/test/pipeline/test-nginx.log-expected.json b/packages/nginx/data_stream/access/_dev/test/pipeline/test-nginx.log-expected.json index d0b3f9d76fa..9b7e7cc7ae4 100644 --- a/packages/nginx/data_stream/access/_dev/test/pipeline/test-nginx.log-expected.json +++ b/packages/nginx/data_stream/access/_dev/test/pipeline/test-nginx.log-expected.json @@ -23,7 +23,7 @@ ], "@timestamp": "2016-12-07T10:05:07.000Z", "ecs": { - "version": "1.12.0" + "version": "8.0.0" }, "_tmp": {}, "related": { @@ -33,7 +33,7 @@ }, "http": { "request": { - "method": "get" + "method": "GET" }, "version": "1.1", "response": { @@ -44,7 +44,7 @@ } }, "event": { - "ingested": "2021-12-14T14:48:34.382243946Z", + "ingested": "2022-01-12T03:18:18.817425179Z", "original": "10.0.0.2, 10.0.0.1, 127.0.0.1 - - [07/Dec/2016:11:05:07 +0100] \"GET /ocelot HTTP/1.1\" 200 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -91,7 +91,7 @@ ], "@timestamp": "2017-05-29T19:02:48.000Z", "ecs": { - "version": "1.12.0" + "version": "8.0.0" }, "_tmp": {}, "related": { @@ -101,7 +101,7 @@ }, "http": { "request": { - "method": "get" + "method": "GET" }, "version": "1.1", "response": { @@ -112,7 +112,7 @@ } }, "event": { - "ingested": "2021-12-14T14:48:34.382246417Z", + "ingested": "2022-01-12T03:18:18.817428385Z", "original": "172.17.0.1 - - [29/May/2017:19:02:48 +0000] \"GET /stringpatch HTTP/1.1\" 404 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -173,7 +173,7 @@ ], "@timestamp": "2016-12-07T10:05:07.000Z", "ecs": { - "version": "1.12.0" + "version": "8.0.0" }, "_tmp": {}, "related": { @@ -183,7 +183,7 @@ }, "http": { "request": { - "method": "get" + "method": "GET" }, "version": "1.1", "response": { @@ -194,7 +194,7 @@ } }, "event": { - "ingested": "2021-12-14T14:48:34.382246878Z", + "ingested": "2022-01-12T03:18:18.817429582Z", "original": "10.0.0.2, 10.0.0.1, 67.43.156.14 - - [07/Dec/2016:11:05:07 +0100] \"GET /ocelot HTTP/1.1\" 200 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -253,7 +253,7 @@ ], "@timestamp": "2016-12-07T10:05:07.000Z", "ecs": { - "version": "1.12.0" + "version": "8.0.0" }, "_tmp": {}, "related": { @@ -263,7 +263,7 @@ }, "http": { "request": { - "method": "get" + "method": "GET" }, "version": "1.1", "response": { @@ -274,7 +274,7 @@ } }, "event": { - "ingested": "2021-12-14T14:48:34.382247234Z", + "ingested": "2022-01-12T03:18:18.817430662Z", "original": "67.43.156.14 - - [07/Dec/2016:11:05:07 +0100] \"GET /ocelot HTTP/1.1\" 200 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36\"\n\"10.5.102.222, 199.96.1.1, 204.246.1.1\" 10.2.1.185 - - [22/Jan/2016:13:18:29 +0000] \"GET /assets/xxxx?q=100 HTTP/1.1\" 200 25507 \"-\" \"Amazon CloudFront\"\n2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6, 10.225.192.17 10.2.2.121 - - [30/Dec/2016:06:47:09 +0000] \"GET /test.html HTTP/1.1\" 404 8571 \"-\" \"Mozilla/5.0 (compatible; Facebot 1.0; https://developers.facebook.com/docs/sharing/webmasters/crawler)\"", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -303,7 +303,7 @@ { "@timestamp": "2018-04-12T07:48:40.000Z", "ecs": { - "version": "1.12.0" + "version": "8.0.0" }, "nginx": { "access": { @@ -330,7 +330,7 @@ "ip": "127.0.0.1" }, "event": { - "ingested": "2021-12-14T14:48:34.382247609Z", + "ingested": "2022-01-12T03:18:18.817431696Z", "original": "127.0.0.1 - - [12/Apr/2018:09:48:40 +0200] \"\" 400 0 \"-\" \"-\"\nunix: - - [26/Feb/2019:15:39:42 +0100] \"hello\" 400 173 \"-\" \"-\"\nlocalhost - - [29/May/2017:19:02:48 +0000] \"GET /test2 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"\nlocalhost, localhost - - [29/May/2017:19:02:48 +0000] \"GET /test2 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"\n", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/nginx/data_stream/access/_dev/test/pipeline/test-test-with-host.log-expected.json b/packages/nginx/data_stream/access/_dev/test/pipeline/test-test-with-host.log-expected.json index f829c774f06..ed86f874662 100644 --- a/packages/nginx/data_stream/access/_dev/test/pipeline/test-test-with-host.log-expected.json +++ b/packages/nginx/data_stream/access/_dev/test/pipeline/test-test-with-host.log-expected.json @@ -27,7 +27,7 @@ ], "@timestamp": "2016-12-07T10:05:07.000Z", "ecs": { - "version": "1.12.0" + "version": "8.0.0" }, "_tmp": {}, "related": { @@ -37,7 +37,7 @@ }, "http": { "request": { - "method": "get" + "method": "GET" }, "version": "1.1", "response": { @@ -48,7 +48,7 @@ } }, "event": { - "ingested": "2021-12-14T14:48:35.072680631Z", + "ingested": "2022-01-12T03:18:20.996551626Z", "original": "example.com 10.0.0.2, 10.0.0.1, 127.0.0.1 - - [07/Dec/2016:11:05:07 +0100] \"GET /ocelot HTTP/1.1\" 200 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"\nexample.com 172.17.0.1 - - [29/May/2017:19:02:48 +0000] \"GET /stringpatch HTTP/1.1\" 404 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"\nexample.com 10.0.0.2, 10.0.0.1, 67.43.156.14 - - [07/Dec/2016:11:05:07 +0100] \"GET /ocelot HTTP/1.1\" 200 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"\nexample.com:80 67.43.156.14 - - [07/Dec/2016:11:05:07 +0100] \"GET /ocelot HTTP/1.1\" 200 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36\"\nexample.com:80 \"10.5.102.222, 199.96.1.1, 204.246.1.1\" 10.2.1.185 - - [22/Jan/2016:13:18:29 +0000] \"GET /assets/xxxx?q=100 HTTP/1.1\" 200 25507 \"-\" \"Amazon CloudFront\"", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -110,7 +110,7 @@ ], "@timestamp": "2016-12-30T06:47:09.000Z", "ecs": { - "version": "1.12.0" + "version": "8.0.0" }, "_tmp": {}, "related": { @@ -121,7 +121,7 @@ }, "http": { "request": { - "method": "get" + "method": "GET" }, "version": "1.1", "response": { @@ -132,7 +132,7 @@ } }, "event": { - "ingested": "2021-12-14T14:48:35.072682974Z", + "ingested": "2022-01-12T03:18:20.996554674Z", "original": "67.43.156.15 2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6, 10.225.192.17 10.2.2.121 - - [30/Dec/2016:06:47:09 +0000] \"GET /test.html HTTP/1.1\" 404 8571 \"-\" \"Mozilla/5.0 (compatible; Facebot 1.0; https://developers.facebook.com/docs/sharing/webmasters/crawler)\"", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -156,7 +156,7 @@ { "@timestamp": "2018-04-12T07:48:40.000Z", "ecs": { - "version": "1.12.0" + "version": "8.0.0" }, "nginx": { "access": { @@ -188,7 +188,7 @@ "ip": "127.0.0.1" }, "event": { - "ingested": "2021-12-14T14:48:35.072683446Z", + "ingested": "2022-01-12T03:18:20.996555836Z", "original": "67.43.156.15:80 127.0.0.1 - - [12/Apr/2018:09:48:40 +0200] \"\" 400 0 \"-\" \"-\"\nexample.com:80 unix: - - [26/Feb/2019:15:39:42 +0100] \"hello\" 400 173 \"-\" \"-\"", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -227,7 +227,7 @@ ], "@timestamp": "2017-05-29T19:02:48.000Z", "ecs": { - "version": "1.12.0" + "version": "8.0.0" }, "_tmp": {}, "related": { @@ -237,7 +237,7 @@ }, "http": { "request": { - "method": "get" + "method": "GET" }, "version": "1.1", "response": { @@ -248,7 +248,7 @@ } }, "event": { - "ingested": "2021-12-14T14:48:35.072683803Z", + "ingested": "2022-01-12T03:18:20.996556865Z", "original": "67.43.156.15 localhost - - [29/May/2017:19:02:48 +0000] \"GET /test2 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"\nexample.com localhost, localhost - - [29/May/2017:19:02:48 +0000] \"GET /test2 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/nginx/data_stream/access/elasticsearch/ingest_pipeline/default.yml b/packages/nginx/data_stream/access/elasticsearch/ingest_pipeline/default.yml index 054d992be64..3d35b3955df 100644 --- a/packages/nginx/data_stream/access/elasticsearch/ingest_pipeline/default.yml +++ b/packages/nginx/data_stream/access/elasticsearch/ingest_pipeline/default.yml @@ -10,7 +10,7 @@ processors: value: '{{_ingest.timestamp}}' - set: field: ecs.version - value: '1.12.0' + value: '8.0.0' - rename: field: message target_field: event.original @@ -176,9 +176,6 @@ processors: field: event.outcome value: failure if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400" - - lowercase: - field: http.request.method - ignore_missing: true - append: field: related.ip value: "{{source.ip}}" diff --git a/packages/nginx/data_stream/access/sample_event.json b/packages/nginx/data_stream/access/sample_event.json index b44215034b4..a51a155283c 100644 --- a/packages/nginx/data_stream/access/sample_event.json +++ b/packages/nginx/data_stream/access/sample_event.json @@ -1,106 +1,113 @@ { + "@timestamp": "2022-01-12T03:18:38.000Z", + "_tmp": {}, "agent": { - "hostname": "a73e7856c209", - "name": "a73e7856c209", - "id": "3987d2b3-b40a-4aa0-99fc-478f9d7079ea", - "ephemeral_id": "6d41da1c-5f71-4bd4-b326-a8913bfaa884", + "ephemeral_id": "4056dd2e-500d-40c2-8e0d-353f6c75d828", + "id": "9878d192-22ad-49b6-a6c2-9959b0815d04", + "name": "docker-fleet-agent", "type": "filebeat", - "version": "7.11.0" + "version": "8.0.0-beta1" }, - "nginx": { - "access": { - "remote_ip_list": [ - "127.0.0.1" - ] - } + "data_stream": { + "dataset": "nginx.access", + "namespace": "ep", + "type": "logs" }, - "log": { - "file": { - "path": "/tmp/service_logs/access.log" - }, - "offset": 0 + "ecs": { + "version": "8.0.0" }, "elastic_agent": { - "id": "5ca3af72-37c3-48b6-92e8-176d154bb66f", - "version": "7.11.0", - "snapshot": true - }, - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, - "url": { - "original": "/server-status" - }, - "input": { - "type": "log" - }, - "@timestamp": "2020-12-03T11:41:57.000Z", - "ecs": { - "version": "1.6.0" + "id": "9878d192-22ad-49b6-a6c2-9959b0815d04", + "snapshot": false, + "version": "8.0.0-beta1" }, - "related": { - "ip": [ - "127.0.0.1" + "event": { + "agent_id_status": "verified", + "category": [ + "web" + ], + "created": "2022-01-12T03:19:08.403Z", + "dataset": "nginx.access", + "ingested": "2022-01-12T03:19:09Z", + "kind": "event", + "outcome": "success", + "timezone": "+00:00", + "type": [ + "access" ] }, - "data_stream": { - "namespace": "ep", - "type": "logs", - "dataset": "nginx.access" - }, "host": { - "hostname": "a73e7856c209", - "os": { - "kernel": "4.9.184-linuxkit", - "codename": "Core", - "name": "CentOS Linux", - "family": "redhat", - "version": "7 (Core)", - "platform": "centos" - }, + "architecture": "x86_64", "containerized": true, + "hostname": "docker-fleet-agent", + "id": "4ccba669f0df47fa3f57a9e4169ae7f1", "ip": [ - "192.168.80.6" + "172.18.0.4" ], - "name": "a73e7856c209", - "id": "06c26569966fd125c15acac5d7feffb6", "mac": [ - "02:42:c0:a8:50:06" + "02:42:ac:12:00:04" ], - "architecture": "x86_64" + "name": "docker-fleet-agent", + "os": { + "codename": "Core", + "family": "redhat", + "kernel": "5.11.0-44-generic", + "name": "CentOS Linux", + "platform": "centos", + "type": "linux", + "version": "7 (Core)" + } }, "http": { "request": { - "method": "get" + "method": "GET" }, "response": { - "status_code": 200, "body": { "bytes": 97 - } + }, + "status_code": 200 }, "version": "1.1" }, - "event": { - "timezone": "+00:00", - "created": "2020-12-03T11:42:17.116Z", - "kind": "event", - "category": [ - "web" - ], - "type": [ - "access" - ], - "dataset": "nginx.access", - "outcome": "success" + "input": { + "type": "log" + }, + "log": { + "file": { + "path": "/tmp/service_logs/access.log" + }, + "offset": 0 + }, + "nginx": { + "access": { + "remote_ip_list": [ + "127.0.0.1" + ] + } + }, + "related": { + "ip": [ + "127.0.0.1" + ] + }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "tags": [ + "nginx-access" + ], + "url": { + "original": "/server-status", + "path": "/server-status" }, "user_agent": { - "original": "curl/7.64.0", - "name": "curl", "device": { "name": "Other" }, + "name": "curl", + "original": "curl/7.64.0", "version": "7.64.0" } } \ No newline at end of file diff --git a/packages/nginx/data_stream/error/_dev/test/pipeline/test-error-raw.log-expected.json b/packages/nginx/data_stream/error/_dev/test/pipeline/test-error-raw.log-expected.json index ad761ec0957..6fba8b2b265 100644 --- a/packages/nginx/data_stream/error/_dev/test/pipeline/test-error-raw.log-expected.json +++ b/packages/nginx/data_stream/error/_dev/test/pipeline/test-error-raw.log-expected.json @@ -9,7 +9,7 @@ }, "@timestamp": "2016-10-25T14:49:34.000Z", "ecs": { - "version": "1.12.0" + "version": "8.0.0" }, "nginx": { "error": { @@ -20,7 +20,7 @@ "level": "error" }, "event": { - "ingested": "2021-12-14T14:48:35.787382431Z", + "ingested": "2022-01-12T03:18:22.874276994Z", "original": "2016/10/25 14:49:34 [error] 54053#0: *1 open() \"/usr/local/Cellar/nginx/1.10.2_1/html/favicon.ico\" failed (2: No such file or directory), client: 127.0.0.1, server: localhost, request: \"GET /favicon.ico HTTP/1.1\", host: \"localhost:8080\", referrer: \"http://localhost:8080/\"", "category": [ "web" @@ -45,7 +45,7 @@ }, "@timestamp": "2016-10-25T14:50:44.000Z", "ecs": { - "version": "1.12.0" + "version": "8.0.0" }, "nginx": { "error": { @@ -56,7 +56,7 @@ "level": "error" }, "event": { - "ingested": "2021-12-14T14:48:35.787384966Z", + "ingested": "2022-01-12T03:18:22.874279040Z", "original": "2016/10/25 14:50:44 [error] 54053#0: *3 open() \"/usr/local/Cellar/nginx/1.10.2_1/html/adsasd\" failed (2: No such file or directory), client: 127.0.0.1, server: localhost, request: \"GET /adsasd HTTP/1.1\", host: \"localhost:8080\"", "category": [ "web" @@ -81,7 +81,7 @@ }, "@timestamp": "2019-10-30T23:26:34.000Z", "ecs": { - "version": "1.12.0" + "version": "8.0.0" }, "nginx": { "error": { @@ -92,7 +92,7 @@ "level": "error" }, "event": { - "ingested": "2021-12-14T14:48:35.787385442Z", + "ingested": "2022-01-12T03:18:22.874280016Z", "original": "2019/10/30 23:26:34 [error] 205860#205860: *180289 FastCGI sent in stderr: \"PHP message: PHP Warning: Declaration of FEE_Field_Terms::wrap($content, $taxonomy, $before, $sep, $after) should be compatible with FEE_Field_Post::wrap($content, $post_id = 0) in /var/www/xxx/web/wp-content/plugins/front-end-editor/php/fields/post.php on line 0\nPHP message: PHP Warning: Declaration of FEE_Field_Tags::wrap($content, $before, $sep, $after) should be compatible with FEE_Field_Terms::wrap($content, $taxonomy, $before, $sep, $after) in /var/www/xxx/web/wp-content/plugins/front-end-editor/php/fields/post.php on line 0\nPHP message: PHP Warning: Declaration of FEE_Field_Category::wrap($content, $sep, $parents) should be compatible with FEE_Field_Terms::wrap($content, $taxonomy, $before, $sep, $after) in /var/www/xxx/web/wp-content/plugins/front-end-editor/php/fields/post.php on line 0", "category": [ "web" @@ -117,7 +117,7 @@ }, "@timestamp": "2019-11-05T14:50:44.000Z", "ecs": { - "version": "1.12.0" + "version": "8.0.0" }, "nginx": { "error": { @@ -128,7 +128,7 @@ "level": "error" }, "event": { - "ingested": "2021-12-14T14:48:35.787385877Z", + "ingested": "2022-01-12T03:18:22.874280869Z", "original": "2019/11/05 14:50:44 [error] 54053#0: *3 open() \"/usr/local/Cellar/nginx/1.10.2_1/html/adsasd\" failed (2: No such file or directory), client: 127.0.0.1, server: localhost, request: \"GET /pysio HTTP/1.1\", host: \"localhost:8080\"", "category": [ "web" diff --git a/packages/nginx/data_stream/error/elasticsearch/ingest_pipeline/default.yml b/packages/nginx/data_stream/error/elasticsearch/ingest_pipeline/default.yml index 9ac85a34e9b..1db221d4a4f 100644 --- a/packages/nginx/data_stream/error/elasticsearch/ingest_pipeline/default.yml +++ b/packages/nginx/data_stream/error/elasticsearch/ingest_pipeline/default.yml @@ -9,7 +9,7 @@ processors: value: '{{_ingest.timestamp}}' - set: field: ecs.version - value: '1.12.0' + value: '8.0.0' - rename: field: message target_field: event.original diff --git a/packages/nginx/data_stream/error/sample_event.json b/packages/nginx/data_stream/error/sample_event.json index b832f1b0723..b3323ab8686 100644 --- a/packages/nginx/data_stream/error/sample_event.json +++ b/packages/nginx/data_stream/error/sample_event.json @@ -1,77 +1,82 @@ { + "@timestamp": "2022-01-12T03:19:41.000Z", "agent": { - "hostname": "a73e7856c209", - "name": "a73e7856c209", - "id": "3987d2b3-b40a-4aa0-99fc-478f9d7079ea", - "ephemeral_id": "6d41da1c-5f71-4bd4-b326-a8913bfaa884", + "ephemeral_id": "49a8eb72-7a5e-4584-821f-b26b95b52624", + "id": "9878d192-22ad-49b6-a6c2-9959b0815d04", + "name": "docker-fleet-agent", "type": "filebeat", - "version": "7.11.0" + "version": "8.0.0-beta1" }, - "process": { - "pid": 1, - "thread": { - "id": 1 - } - }, - "nginx": { - "error": {} + "data_stream": { + "dataset": "nginx.error", + "namespace": "ep", + "type": "logs" }, - "log": { - "file": { - "path": "/tmp/service_logs/error.log" - }, - "offset": 0, - "level": "warn" + "ecs": { + "version": "8.0.0" }, "elastic_agent": { - "id": "5ca3af72-37c3-48b6-92e8-176d154bb66f", - "version": "7.11.0", - "snapshot": true - }, - "message": "conflicting server name \"localhost\" on 0.0.0.0:80, ignored", - "input": { - "type": "log" + "id": "9878d192-22ad-49b6-a6c2-9959b0815d04", + "snapshot": false, + "version": "8.0.0-beta1" }, - "@timestamp": "2020-12-03T11:44:39.000Z", - "ecs": { - "version": "1.6.0" - }, - "data_stream": { - "namespace": "ep", - "type": "logs", - "dataset": "nginx.error" + "event": { + "agent_id_status": "verified", + "category": [ + "web" + ], + "created": "2022-01-12T03:20:00.439Z", + "dataset": "nginx.error", + "ingested": "2022-01-12T03:20:06Z", + "kind": "event", + "timezone": "+00:00", + "type": [ + "error" + ] }, "host": { - "hostname": "a73e7856c209", - "os": { - "kernel": "4.9.184-linuxkit", - "codename": "Core", - "name": "CentOS Linux", - "family": "redhat", - "version": "7 (Core)", - "platform": "centos" - }, + "architecture": "x86_64", "containerized": true, + "hostname": "docker-fleet-agent", + "id": "4ccba669f0df47fa3f57a9e4169ae7f1", "ip": [ - "192.168.80.6" + "172.18.0.4" ], - "name": "a73e7856c209", - "id": "06c26569966fd125c15acac5d7feffb6", "mac": [ - "02:42:c0:a8:50:06" + "02:42:ac:12:00:04" ], - "architecture": "x86_64" + "name": "docker-fleet-agent", + "os": { + "codename": "Core", + "family": "redhat", + "kernel": "5.11.0-44-generic", + "name": "CentOS Linux", + "platform": "centos", + "type": "linux", + "version": "7 (Core)" + } }, - "event": { - "timezone": "+00:00", - "created": "2020-12-03T11:44:52.803Z", - "kind": "event", - "category": [ - "web" - ], - "type": [ - "error" - ], - "dataset": "nginx.error" - } + "input": { + "type": "log" + }, + "log": { + "file": { + "path": "/tmp/service_logs/error.log" + }, + "level": "warn", + "offset": 0 + }, + "message": "conflicting server name \"localhost\" on 0.0.0.0:80, ignored", + "nginx": { + "error": {} + }, + "process": { + "pid": 1, + "thread": { + "id": 1 + } + }, + "tags": [ + "nginx-error" + ] } \ No newline at end of file diff --git a/packages/nginx/data_stream/stubstatus/sample_event.json b/packages/nginx/data_stream/stubstatus/sample_event.json index 2f8a9bd9bff..2c90beafcab 100644 --- a/packages/nginx/data_stream/stubstatus/sample_event.json +++ b/packages/nginx/data_stream/stubstatus/sample_event.json @@ -1,72 +1,74 @@ { - "@timestamp": "2020-12-03T11:47:31.996Z", + "@timestamp": "2022-01-12T03:20:44.909Z", + "agent": { + "ephemeral_id": "8f176291-ce69-4319-bca2-af6b2dde74c5", + "id": "9878d192-22ad-49b6-a6c2-9959b0815d04", + "name": "docker-fleet-agent", + "type": "metricbeat", + "version": "8.0.0-beta1" + }, + "data_stream": { + "dataset": "nginx.stubstatus", + "namespace": "ep", + "type": "metrics" + }, + "ecs": { + "version": "1.12.0" + }, + "elastic_agent": { + "id": "9878d192-22ad-49b6-a6c2-9959b0815d04", + "snapshot": false, + "version": "8.0.0-beta1" + }, + "event": { + "agent_id_status": "verified", + "dataset": "nginx.stubstatus", + "duration": 1633671, + "ingested": "2022-01-12T03:20:46Z", + "module": "nginx" + }, "host": { - "hostname": "a73e7856c209", "architecture": "x86_64", - "os": { - "codename": "Core", - "platform": "centos", - "version": "7 (Core)", - "family": "redhat", - "name": "CentOS Linux", - "kernel": "4.9.184-linuxkit" - }, - "name": "a73e7856c209", - "id": "06c26569966fd125c15acac5d7feffb6", "containerized": true, + "hostname": "docker-fleet-agent", + "id": "4ccba669f0df47fa3f57a9e4169ae7f1", "ip": [ - "192.168.80.6" + "172.18.0.4" ], "mac": [ - "02:42:c0:a8:50:06" - ] + "02:42:ac:12:00:04" + ], + "name": "docker-fleet-agent", + "os": { + "codename": "Core", + "family": "redhat", + "kernel": "5.11.0-44-generic", + "name": "CentOS Linux", + "platform": "centos", + "type": "linux", + "version": "7 (Core)" + } }, - "service": { - "type": "nginx", - "address": "http://elastic-package-service_nginx_1:80/server-status" + "metricset": { + "name": "stubstatus", + "period": 10000 }, "nginx": { "stubstatus": { - "requests": 13, - "waiting": 0, - "hostname": "elastic-package-service_nginx_1:80", - "accepts": 13, - "handled": 13, - "current": 13, - "dropped": 0, - "writing": 1, + "accepts": 18, "active": 1, - "reading": 0 + "current": 18, + "dropped": 0, + "handled": 18, + "hostname": "elastic-package-service-nginx-1:80", + "reading": 0, + "requests": 18, + "waiting": 0, + "writing": 1 } }, - "elastic_agent": { - "snapshot": true, - "version": "7.11.0", - "id": "5ca3af72-37c3-48b6-92e8-176d154bb66f" - }, - "ecs": { - "version": "1.6.0" - }, - "event": { - "dataset": "nginx.stubstatus", - "module": "nginx", - "duration": 2231100 - }, - "metricset": { - "period": 10000, - "name": "stubstatus" - }, - "data_stream": { - "type": "metrics", - "dataset": "nginx.stubstatus", - "namespace": "ep" - }, - "agent": { - "type": "metricbeat", - "version": "7.11.0", - "hostname": "a73e7856c209", - "ephemeral_id": "1fbb4215-4ba3-42fa-9984-244b112c9a17", - "id": "2689a72c-6e18-45fe-b493-af1ec86af2b3", - "name": "a73e7856c209" + "service": { + "address": "http://elastic-package-service-nginx-1:80/server-status", + "type": "nginx" } } \ No newline at end of file diff --git a/packages/nginx/docs/README.md b/packages/nginx/docs/README.md index 53ebb724c72..c0883e643eb 100644 --- a/packages/nginx/docs/README.md +++ b/packages/nginx/docs/README.md @@ -158,7 +158,7 @@ An example event for `access` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.domain | Destination domain. | keyword | +| destination.domain | The domain name of the destination system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | | destination.ip | IP address of the destination (IPv4 or IPv6). | ip | | destination.port | Port of the destination. | long | | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | @@ -181,7 +181,7 @@ An example event for `access` looks as following: | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| http.request.method | HTTP request method. Prior to ECS 1.6.0 the following guidance was provided: "The field value must be normalized to lowercase for querying." As of ECS 1.6.0, the guidance is deprecated because the original case of the method may be useful in anomaly detection. Original case will be mandated in ECS 2.0.0 | keyword | +| http.request.method | HTTP request method. The value should retain its casing from the original event. For example, `GET`, `get`, and `GeT` are all considered valid values for this field. | keyword | | http.request.referrer | Referrer for this HTTP request. | keyword | | http.response.body.bytes | Size in bytes of the response body. | long | | http.response.status_code | HTTP response status code. | long | diff --git a/packages/nginx/manifest.yml b/packages/nginx/manifest.yml index 654d7e24c39..84579a3bc0a 100644 --- a/packages/nginx/manifest.yml +++ b/packages/nginx/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: nginx title: Nginx -version: 1.2.3 +version: 1.3.0 license: basic description: Collect logs and metrics from Nginx HTTP servers with Elastic Agent. type: integration From 87cdc001b0ab0d8315958b6198bcab7683122e76 Mon Sep 17 00:00:00 2001 From: Sai Kiran <85323324+r00tu53r@users.noreply.github.com> Date: Wed, 12 Jan 2022 16:53:58 +1100 Subject: [PATCH 2/3] fix changelog --- packages/nginx/changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/nginx/changelog.yml b/packages/nginx/changelog.yml index e0754578df7..03746542d33 100644 --- a/packages/nginx/changelog.yml +++ b/packages/nginx/changelog.yml @@ -1,9 +1,9 @@ +# newer versions go on top - version: "1.3.0" changes: - description: Update to ECS 8.0 type: enhancement link: https://github.com/elastic/integrations/pull/2505 -# newer versions go on top - version: "1.2.3" changes: - description: Regenerate test files using the new GeoIP database From 6d82c028839e24b090296d1d0b744e31fdedce35 Mon Sep 17 00:00:00 2001 From: Sai Kiran <85323324+r00tu53r@users.noreply.github.com> Date: Wed, 12 Jan 2022 17:21:13 +1100 Subject: [PATCH 3/3] update readme --- packages/nginx/docs/README.md | 390 ++++++++++++++++++---------------- 1 file changed, 202 insertions(+), 188 deletions(-) diff --git a/packages/nginx/docs/README.md b/packages/nginx/docs/README.md index c0883e643eb..301a3d1f42d 100644 --- a/packages/nginx/docs/README.md +++ b/packages/nginx/docs/README.md @@ -30,108 +30,115 @@ An example event for `access` looks as following: ```json { + "@timestamp": "2022-01-12T03:18:38.000Z", + "_tmp": {}, "agent": { - "hostname": "a73e7856c209", - "name": "a73e7856c209", - "id": "3987d2b3-b40a-4aa0-99fc-478f9d7079ea", - "ephemeral_id": "6d41da1c-5f71-4bd4-b326-a8913bfaa884", + "ephemeral_id": "4056dd2e-500d-40c2-8e0d-353f6c75d828", + "id": "9878d192-22ad-49b6-a6c2-9959b0815d04", + "name": "docker-fleet-agent", "type": "filebeat", - "version": "7.11.0" + "version": "8.0.0-beta1" }, - "nginx": { - "access": { - "remote_ip_list": [ - "127.0.0.1" - ] - } + "data_stream": { + "dataset": "nginx.access", + "namespace": "ep", + "type": "logs" }, - "log": { - "file": { - "path": "/tmp/service_logs/access.log" - }, - "offset": 0 + "ecs": { + "version": "8.0.0" }, "elastic_agent": { - "id": "5ca3af72-37c3-48b6-92e8-176d154bb66f", - "version": "7.11.0", - "snapshot": true - }, - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, - "url": { - "original": "/server-status" + "id": "9878d192-22ad-49b6-a6c2-9959b0815d04", + "snapshot": false, + "version": "8.0.0-beta1" }, - "input": { - "type": "log" - }, - "@timestamp": "2020-12-03T11:41:57.000Z", - "ecs": { - "version": "1.6.0" - }, - "related": { - "ip": [ - "127.0.0.1" + "event": { + "agent_id_status": "verified", + "category": [ + "web" + ], + "created": "2022-01-12T03:19:08.403Z", + "dataset": "nginx.access", + "ingested": "2022-01-12T03:19:09Z", + "kind": "event", + "outcome": "success", + "timezone": "+00:00", + "type": [ + "access" ] }, - "data_stream": { - "namespace": "ep", - "type": "logs", - "dataset": "nginx.access" - }, "host": { - "hostname": "a73e7856c209", - "os": { - "kernel": "4.9.184-linuxkit", - "codename": "Core", - "name": "CentOS Linux", - "family": "redhat", - "version": "7 (Core)", - "platform": "centos" - }, + "architecture": "x86_64", "containerized": true, + "hostname": "docker-fleet-agent", + "id": "4ccba669f0df47fa3f57a9e4169ae7f1", "ip": [ - "192.168.80.6" + "172.18.0.4" ], - "name": "a73e7856c209", - "id": "06c26569966fd125c15acac5d7feffb6", "mac": [ - "02:42:c0:a8:50:06" + "02:42:ac:12:00:04" ], - "architecture": "x86_64" + "name": "docker-fleet-agent", + "os": { + "codename": "Core", + "family": "redhat", + "kernel": "5.11.0-44-generic", + "name": "CentOS Linux", + "platform": "centos", + "type": "linux", + "version": "7 (Core)" + } }, "http": { "request": { - "method": "get" + "method": "GET" }, "response": { - "status_code": 200, "body": { "bytes": 97 - } + }, + "status_code": 200 }, "version": "1.1" }, - "event": { - "timezone": "+00:00", - "created": "2020-12-03T11:42:17.116Z", - "kind": "event", - "category": [ - "web" - ], - "type": [ - "access" - ], - "dataset": "nginx.access", - "outcome": "success" + "input": { + "type": "log" + }, + "log": { + "file": { + "path": "/tmp/service_logs/access.log" + }, + "offset": 0 + }, + "nginx": { + "access": { + "remote_ip_list": [ + "127.0.0.1" + ] + } + }, + "related": { + "ip": [ + "127.0.0.1" + ] + }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "tags": [ + "nginx-access" + ], + "url": { + "original": "/server-status", + "path": "/server-status" }, "user_agent": { - "original": "curl/7.64.0", - "name": "curl", "device": { "name": "Other" }, + "name": "curl", + "original": "curl/7.64.0", "version": "7.64.0" } } @@ -227,81 +234,86 @@ An example event for `error` looks as following: ```json { + "@timestamp": "2022-01-12T03:19:41.000Z", "agent": { - "hostname": "a73e7856c209", - "name": "a73e7856c209", - "id": "3987d2b3-b40a-4aa0-99fc-478f9d7079ea", - "ephemeral_id": "6d41da1c-5f71-4bd4-b326-a8913bfaa884", + "ephemeral_id": "49a8eb72-7a5e-4584-821f-b26b95b52624", + "id": "9878d192-22ad-49b6-a6c2-9959b0815d04", + "name": "docker-fleet-agent", "type": "filebeat", - "version": "7.11.0" + "version": "8.0.0-beta1" }, - "process": { - "pid": 1, - "thread": { - "id": 1 - } - }, - "nginx": { - "error": {} + "data_stream": { + "dataset": "nginx.error", + "namespace": "ep", + "type": "logs" }, - "log": { - "file": { - "path": "/tmp/service_logs/error.log" - }, - "offset": 0, - "level": "warn" + "ecs": { + "version": "8.0.0" }, "elastic_agent": { - "id": "5ca3af72-37c3-48b6-92e8-176d154bb66f", - "version": "7.11.0", - "snapshot": true + "id": "9878d192-22ad-49b6-a6c2-9959b0815d04", + "snapshot": false, + "version": "8.0.0-beta1" }, - "message": "conflicting server name \"localhost\" on 0.0.0.0:80, ignored", - "input": { - "type": "log" - }, - "@timestamp": "2020-12-03T11:44:39.000Z", - "ecs": { - "version": "1.6.0" - }, - "data_stream": { - "namespace": "ep", - "type": "logs", - "dataset": "nginx.error" + "event": { + "agent_id_status": "verified", + "category": [ + "web" + ], + "created": "2022-01-12T03:20:00.439Z", + "dataset": "nginx.error", + "ingested": "2022-01-12T03:20:06Z", + "kind": "event", + "timezone": "+00:00", + "type": [ + "error" + ] }, "host": { - "hostname": "a73e7856c209", - "os": { - "kernel": "4.9.184-linuxkit", - "codename": "Core", - "name": "CentOS Linux", - "family": "redhat", - "version": "7 (Core)", - "platform": "centos" - }, + "architecture": "x86_64", "containerized": true, + "hostname": "docker-fleet-agent", + "id": "4ccba669f0df47fa3f57a9e4169ae7f1", "ip": [ - "192.168.80.6" + "172.18.0.4" ], - "name": "a73e7856c209", - "id": "06c26569966fd125c15acac5d7feffb6", "mac": [ - "02:42:c0:a8:50:06" + "02:42:ac:12:00:04" ], - "architecture": "x86_64" + "name": "docker-fleet-agent", + "os": { + "codename": "Core", + "family": "redhat", + "kernel": "5.11.0-44-generic", + "name": "CentOS Linux", + "platform": "centos", + "type": "linux", + "version": "7 (Core)" + } }, - "event": { - "timezone": "+00:00", - "created": "2020-12-03T11:44:52.803Z", - "kind": "event", - "category": [ - "web" - ], - "type": [ - "error" - ], - "dataset": "nginx.error" - } + "input": { + "type": "log" + }, + "log": { + "file": { + "path": "/tmp/service_logs/error.log" + }, + "level": "warn", + "offset": 0 + }, + "message": "conflicting server name \"localhost\" on 0.0.0.0:80, ignored", + "nginx": { + "error": {} + }, + "process": { + "pid": 1, + "thread": { + "id": 1 + } + }, + "tags": [ + "nginx-error" + ] } ``` @@ -379,75 +391,77 @@ An example event for `stubstatus` looks as following: ```json { - "@timestamp": "2020-12-03T11:47:31.996Z", + "@timestamp": "2022-01-12T03:20:44.909Z", + "agent": { + "ephemeral_id": "8f176291-ce69-4319-bca2-af6b2dde74c5", + "id": "9878d192-22ad-49b6-a6c2-9959b0815d04", + "name": "docker-fleet-agent", + "type": "metricbeat", + "version": "8.0.0-beta1" + }, + "data_stream": { + "dataset": "nginx.stubstatus", + "namespace": "ep", + "type": "metrics" + }, + "ecs": { + "version": "1.12.0" + }, + "elastic_agent": { + "id": "9878d192-22ad-49b6-a6c2-9959b0815d04", + "snapshot": false, + "version": "8.0.0-beta1" + }, + "event": { + "agent_id_status": "verified", + "dataset": "nginx.stubstatus", + "duration": 1633671, + "ingested": "2022-01-12T03:20:46Z", + "module": "nginx" + }, "host": { - "hostname": "a73e7856c209", "architecture": "x86_64", - "os": { - "codename": "Core", - "platform": "centos", - "version": "7 (Core)", - "family": "redhat", - "name": "CentOS Linux", - "kernel": "4.9.184-linuxkit" - }, - "name": "a73e7856c209", - "id": "06c26569966fd125c15acac5d7feffb6", "containerized": true, + "hostname": "docker-fleet-agent", + "id": "4ccba669f0df47fa3f57a9e4169ae7f1", "ip": [ - "192.168.80.6" + "172.18.0.4" ], "mac": [ - "02:42:c0:a8:50:06" - ] + "02:42:ac:12:00:04" + ], + "name": "docker-fleet-agent", + "os": { + "codename": "Core", + "family": "redhat", + "kernel": "5.11.0-44-generic", + "name": "CentOS Linux", + "platform": "centos", + "type": "linux", + "version": "7 (Core)" + } }, - "service": { - "type": "nginx", - "address": "http://elastic-package-service_nginx_1:80/server-status" + "metricset": { + "name": "stubstatus", + "period": 10000 }, "nginx": { "stubstatus": { - "requests": 13, - "waiting": 0, - "hostname": "elastic-package-service_nginx_1:80", - "accepts": 13, - "handled": 13, - "current": 13, - "dropped": 0, - "writing": 1, + "accepts": 18, "active": 1, - "reading": 0 + "current": 18, + "dropped": 0, + "handled": 18, + "hostname": "elastic-package-service-nginx-1:80", + "reading": 0, + "requests": 18, + "waiting": 0, + "writing": 1 } }, - "elastic_agent": { - "snapshot": true, - "version": "7.11.0", - "id": "5ca3af72-37c3-48b6-92e8-176d154bb66f" - }, - "ecs": { - "version": "1.6.0" - }, - "event": { - "dataset": "nginx.stubstatus", - "module": "nginx", - "duration": 2231100 - }, - "metricset": { - "period": 10000, - "name": "stubstatus" - }, - "data_stream": { - "type": "metrics", - "dataset": "nginx.stubstatus", - "namespace": "ep" - }, - "agent": { - "type": "metricbeat", - "version": "7.11.0", - "hostname": "a73e7856c209", - "ephemeral_id": "1fbb4215-4ba3-42fa-9984-244b112c9a17", - "id": "2689a72c-6e18-45fe-b493-af1ec86af2b3", - "name": "a73e7856c209" + "service": { + "address": "http://elastic-package-service-nginx-1:80/server-status", + "type": "nginx" } } ```