cloudflare_logpush: use EdgeStartTimestamp event timestamp (#5599)

elastic · Mar 21, 2023 · 35f9966 · 35f9966
1 parent 50ea10d
commit 35f9966
Show file tree

Hide file tree

Showing 6 changed files with 32 additions and 23 deletions.
diff --git a/packages/cloudflare_logpush/changelog.yml b/packages/cloudflare_logpush/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.6.0"
+  changes:
+    - description: Use `EdgeStartTimestamp` as `@timestamp` time.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/5599
 - version: "0.5.1"
   changes:
     - description: Added categories and/or subcategories.

diff --git a/.../data_stream/http_request/_dev/test/pipeline/test-pipeline-http-request.log-expected.json b/.../data_stream/http_request/_dev/test/pipeline/test-pipeline-http-request.log-expected.json
@@ -1,6 +1,7 @@
 {
     "expected": [
         {
+            "@timestamp": "2022-05-25T13:25:26Z",
             "cloudflare_logpush": {
                 "http_request": {
                     "bot": {
@@ -243,6 +244,7 @@
             }
         },
         {
+            "@timestamp": "2022-05-25T13:25:26Z",
             "cloudflare_logpush": {
                 "http_request": {
                     "bot": {

diff --git a/...ges/cloudflare_logpush/data_stream/http_request/elasticsearch/ingest_pipeline/default.yml b/...ges/cloudflare_logpush/data_stream/http_request/elasticsearch/ingest_pipeline/default.yml
@@ -22,8 +22,8 @@ processors:
       field: event.type
       value: [info]
   - date:
-      field: json.EdgeEndTimestamp
-      if: ctx.json?.EdgeEndTimestamp != null && ctx.json.EdgeEndTimestamp != ''
+      field: json.EdgeStartTimestamp
+      if: ctx.json?.EdgeStartTimestamp != null && ctx.json.EdgeStartTimestamp != ''
       formats:
         - ISO8601
         - uuuu-MM-dd'T'HH:mm:ssX
@@ -32,14 +32,18 @@ processors:
         - yyyy-MM-dd'T'HH:mm:ss.SSSZ
         - UNIX_MS
       timezone: UTC
-      target_field: cloudflare_logpush.http_request.edge.end_time
+      target_field: cloudflare_logpush.http_request.edge.start_time
       on_failure:
         - append:
             field: error.message
             value: '{{{_ingest.on_failure_message}}}'
+  - set:
+      if: ctx.json?.EdgeStartTimestamp != null
+      field: '@timestamp'
+      copy_from: json.EdgeStartTimestamp
   - date:
-      field: json.EdgeStartTimestamp
-      if: ctx.json?.EdgeStartTimestamp != null && ctx.json.EdgeStartTimestamp != ''
+      field: json.EdgeEndTimestamp
+      if: ctx.json?.EdgeEndTimestamp != null && ctx.json.EdgeEndTimestamp != ''
       formats:
         - ISO8601
         - uuuu-MM-dd'T'HH:mm:ssX
@@ -48,7 +52,7 @@ processors:
         - yyyy-MM-dd'T'HH:mm:ss.SSSZ
         - UNIX_MS
       timezone: UTC
-      target_field: cloudflare_logpush.http_request.edge.start_time
+      target_field: cloudflare_logpush.http_request.edge.end_time
       on_failure:
         - append:
             field: error.message

diff --git a/packages/cloudflare_logpush/data_stream/http_request/sample_event.json b/packages/cloudflare_logpush/data_stream/http_request/sample_event.json
@@ -1,12 +1,11 @@
 {
-    "@timestamp": "2022-09-01T10:08:19.901Z",
+    "@timestamp": "2022-05-25T13:25:26Z",
     "agent": {
-        "ephemeral_id": "799a05d5-4523-4df3-8588-0a26bce74843",
-        "hostname": "docker-fleet-agent",
-        "id": "8539930e-8f7a-48ac-af3e-7f098b7d6ea2",
+        "ephemeral_id": "dfdb0a3e-5218-4b1e-8ce1-38ad94902bf6",
+        "id": "8eafc4b3-b5f0-4541-ae2a-c9bb2f2e0074",
         "name": "docker-fleet-agent",
         "type": "filebeat",
-        "version": "7.17.0"
+        "version": "8.6.1"
     },
     "cloudflare_logpush": {
         "http_request": {
@@ -188,17 +187,17 @@
         "version": "8.6.0"
     },
     "elastic_agent": {
-        "id": "8539930e-8f7a-48ac-af3e-7f098b7d6ea2",
+        "id": "8eafc4b3-b5f0-4541-ae2a-c9bb2f2e0074",
         "snapshot": false,
-        "version": "7.17.0"
+        "version": "8.6.1"
     },
     "event": {
         "agent_id_status": "verified",
         "category": [
             "network"
         ],
         "dataset": "cloudflare_logpush.http_request",
-        "ingested": "2022-09-01T10:08:20Z",
+        "ingested": "2023-03-21T00:21:42Z",
         "kind": "event",
         "original": "{\"BotScore\":\"20\",\"BotScoreSrc\":\"Verified Bot\",\"BotTags\":\"bing\",\"CacheCacheStatus\":\"dynamic\",\"CacheResponseBytes\":983828,\"CacheResponseStatus\":200,\"CacheTieredFill\":false,\"ClientASN\":43766,\"ClientCountry\":\"sa\",\"ClientDeviceType\":\"desktop\",\"ClientIP\":\"175.16.199.0\",\"ClientIPClass\":\"noRecord\",\"ClientMTLSAuthCertFingerprint\":\"Fingerprint\",\"ClientMTLSAuthStatus\":\"unknown\",\"ClientRequestBytes\":5800,\"ClientRequestHost\":\"xyz.example.com\",\"ClientRequestMethod\":\"POST\",\"ClientRequestPath\":\"/xyz/checkout\",\"ClientRequestProtocol\":\"HTTP/1.1\",\"ClientRequestReferer\":\"https://example.com/s/example/default?sourcerer=(default:(id:!n,selectedPatterns:!(example,%27logs-endpoint.*-example%27,%27logs-system.*-example%27,%27logs-windows.*-example%27)))\\u0026timerange=(global:(linkTo:!(),timerange:(from:%272022-05-16T06:26:36.340Z%27,fromStr:now-24h,kind:relative,to:%272022-05-17T06:26:36.340Z%27,toStr:now)),timeline:(linkTo:!(),timerange:(from:%272022-04-17T22:00:00.000Z%27,kind:absolute,to:%272022-04-18T21:59:59.999Z%27)))\\u0026timeline=(activeTab:notes,graphEventId:%27%27,id:%279844bdd4-4dd6-5b22-ab40-3cd46fce8d6b%27,isOpen:!t)\",\"ClientRequestScheme\":\"https\",\"ClientRequestSource\":\"edgeWorkerFetch\",\"ClientRequestURI\":\"/s/example/api/telemetry/v2/clusters/_stats\",\"ClientRequestUserAgent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36\",\"ClientSSLCipher\":\"NONE\",\"ClientSSLProtocol\":\"TLSv1.2\",\"ClientSrcPort\":0,\"ClientTCPRTTMs\":0,\"ClientXRequestedWith\":\"Request With\",\"Cookies\":{\"key\":\"value\"},\"EdgeCFConnectingO2O\":false,\"EdgeColoCode\":\"RUH\",\"EdgeColoID\":339,\"EdgeEndTimestamp\":\"2022-05-25T13:25:32Z\",\"EdgePathingOp\":\"wl\",\"EdgePathingSrc\":\"macro\",\"EdgePathingStatus\":\"nr\",\"EdgeRateLimitAction\":\"unknown\",\"EdgeRateLimitID\":0,\"EdgeRequestHost\":\"abc.example.com\",\"EdgeResponseBodyBytes\":980397,\"EdgeResponseBytes\":981308,\"EdgeResponseCompressionRatio\":0,\"EdgeResponseContentType\":\"application/json\",\"EdgeResponseStatus\":200,\"EdgeServerIP\":\"1.128.0.0\",\"EdgeStartTimestamp\":\"2022-05-25T13:25:26Z\",\"EdgeTimeToFirstByteMs\":5333,\"OriginDNSResponseTimeMs\":3,\"OriginIP\":\"67.43.156.0\",\"OriginRequestHeaderSendDurationMs\":0,\"OriginResponseBytes\":0,\"OriginResponseDurationMs\":5319,\"OriginResponseHTTPExpires\":\"2022-05-27T13:25:26Z\",\"OriginResponseHTTPLastModified\":\"2022-05-26T13:25:26Z\",\"OriginResponseHeaderReceiveDurationMs\":5155,\"OriginResponseStatus\":200,\"OriginResponseTime\":5232000000,\"OriginSSLProtocol\":\"TLSv1.2\",\"OriginTCPHandshakeDurationMs\":24,\"OriginTLSHandshakeDurationMs\":53,\"ParentRayID\":\"710e98d93d50357d\",\"RayID\":\"710e98d9367f357d\",\"SecurityLevel\":\"off\",\"SmartRouteColoID\":20,\"UpperTierColoID\":0,\"WAFAction\":\"unknown\",\"WAFFlags\":\"0\",\"WAFMatchedVar\":\"example\",\"WAFProfile\":\"unknown\",\"WAFRuleID\":\"98d93d5\",\"WAFRuleMessage\":\"matchad variable message\",\"WorkerCPUTime\":0,\"WorkerStatus\":\"unknown\",\"WorkerSubrequest\":true,\"WorkerSubrequestCount\":0,\"ZoneID\":393347122,\"ZoneName\":\"example.com\"}",
         "type": [

diff --git a/packages/cloudflare_logpush/docs/README.md b/packages/cloudflare_logpush/docs/README.md
@@ -735,14 +735,13 @@ An example event for `http_request` looks as following:
 
 ```json
 {
-    "@timestamp": "2022-09-01T10:08:19.901Z",
+    "@timestamp": "2022-05-25T13:25:26Z",
     "agent": {
-        "ephemeral_id": "799a05d5-4523-4df3-8588-0a26bce74843",
-        "hostname": "docker-fleet-agent",
-        "id": "8539930e-8f7a-48ac-af3e-7f098b7d6ea2",
+        "ephemeral_id": "dfdb0a3e-5218-4b1e-8ce1-38ad94902bf6",
+        "id": "8eafc4b3-b5f0-4541-ae2a-c9bb2f2e0074",
         "name": "docker-fleet-agent",
         "type": "filebeat",
-        "version": "7.17.0"
+        "version": "8.6.1"
     },
     "cloudflare_logpush": {
         "http_request": {
@@ -924,17 +923,17 @@ An example event for `http_request` looks as following:
         "version": "8.6.0"
     },
     "elastic_agent": {
-        "id": "8539930e-8f7a-48ac-af3e-7f098b7d6ea2",
+        "id": "8eafc4b3-b5f0-4541-ae2a-c9bb2f2e0074",
         "snapshot": false,
-        "version": "7.17.0"
+        "version": "8.6.1"
     },
     "event": {
         "agent_id_status": "verified",
         "category": [
             "network"
         ],
         "dataset": "cloudflare_logpush.http_request",
-        "ingested": "2022-09-01T10:08:20Z",
+        "ingested": "2023-03-21T00:21:42Z",
         "kind": "event",
         "original": "{\"BotScore\":\"20\",\"BotScoreSrc\":\"Verified Bot\",\"BotTags\":\"bing\",\"CacheCacheStatus\":\"dynamic\",\"CacheResponseBytes\":983828,\"CacheResponseStatus\":200,\"CacheTieredFill\":false,\"ClientASN\":43766,\"ClientCountry\":\"sa\",\"ClientDeviceType\":\"desktop\",\"ClientIP\":\"175.16.199.0\",\"ClientIPClass\":\"noRecord\",\"ClientMTLSAuthCertFingerprint\":\"Fingerprint\",\"ClientMTLSAuthStatus\":\"unknown\",\"ClientRequestBytes\":5800,\"ClientRequestHost\":\"xyz.example.com\",\"ClientRequestMethod\":\"POST\",\"ClientRequestPath\":\"/xyz/checkout\",\"ClientRequestProtocol\":\"HTTP/1.1\",\"ClientRequestReferer\":\"https://example.com/s/example/default?sourcerer=(default:(id:!n,selectedPatterns:!(example,%27logs-endpoint.*-example%27,%27logs-system.*-example%27,%27logs-windows.*-example%27)))\\u0026timerange=(global:(linkTo:!(),timerange:(from:%272022-05-16T06:26:36.340Z%27,fromStr:now-24h,kind:relative,to:%272022-05-17T06:26:36.340Z%27,toStr:now)),timeline:(linkTo:!(),timerange:(from:%272022-04-17T22:00:00.000Z%27,kind:absolute,to:%272022-04-18T21:59:59.999Z%27)))\\u0026timeline=(activeTab:notes,graphEventId:%27%27,id:%279844bdd4-4dd6-5b22-ab40-3cd46fce8d6b%27,isOpen:!t)\",\"ClientRequestScheme\":\"https\",\"ClientRequestSource\":\"edgeWorkerFetch\",\"ClientRequestURI\":\"/s/example/api/telemetry/v2/clusters/_stats\",\"ClientRequestUserAgent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36\",\"ClientSSLCipher\":\"NONE\",\"ClientSSLProtocol\":\"TLSv1.2\",\"ClientSrcPort\":0,\"ClientTCPRTTMs\":0,\"ClientXRequestedWith\":\"Request With\",\"Cookies\":{\"key\":\"value\"},\"EdgeCFConnectingO2O\":false,\"EdgeColoCode\":\"RUH\",\"EdgeColoID\":339,\"EdgeEndTimestamp\":\"2022-05-25T13:25:32Z\",\"EdgePathingOp\":\"wl\",\"EdgePathingSrc\":\"macro\",\"EdgePathingStatus\":\"nr\",\"EdgeRateLimitAction\":\"unknown\",\"EdgeRateLimitID\":0,\"EdgeRequestHost\":\"abc.example.com\",\"EdgeResponseBodyBytes\":980397,\"EdgeResponseBytes\":981308,\"EdgeResponseCompressionRatio\":0,\"EdgeResponseContentType\":\"application/json\",\"EdgeResponseStatus\":200,\"EdgeServerIP\":\"1.128.0.0\",\"EdgeStartTimestamp\":\"2022-05-25T13:25:26Z\",\"EdgeTimeToFirstByteMs\":5333,\"OriginDNSResponseTimeMs\":3,\"OriginIP\":\"67.43.156.0\",\"OriginRequestHeaderSendDurationMs\":0,\"OriginResponseBytes\":0,\"OriginResponseDurationMs\":5319,\"OriginResponseHTTPExpires\":\"2022-05-27T13:25:26Z\",\"OriginResponseHTTPLastModified\":\"2022-05-26T13:25:26Z\",\"OriginResponseHeaderReceiveDurationMs\":5155,\"OriginResponseStatus\":200,\"OriginResponseTime\":5232000000,\"OriginSSLProtocol\":\"TLSv1.2\",\"OriginTCPHandshakeDurationMs\":24,\"OriginTLSHandshakeDurationMs\":53,\"ParentRayID\":\"710e98d93d50357d\",\"RayID\":\"710e98d9367f357d\",\"SecurityLevel\":\"off\",\"SmartRouteColoID\":20,\"UpperTierColoID\":0,\"WAFAction\":\"unknown\",\"WAFFlags\":\"0\",\"WAFMatchedVar\":\"example\",\"WAFProfile\":\"unknown\",\"WAFRuleID\":\"98d93d5\",\"WAFRuleMessage\":\"matchad variable message\",\"WorkerCPUTime\":0,\"WorkerStatus\":\"unknown\",\"WorkerSubrequest\":true,\"WorkerSubrequestCount\":0,\"ZoneID\":393347122,\"ZoneName\":\"example.com\"}",
         "type": [

diff --git a/packages/cloudflare_logpush/manifest.yml b/packages/cloudflare_logpush/manifest.yml
@@ -1,7 +1,7 @@
 format_version: 1.0.0
 name: cloudflare_logpush
 title: Cloudflare Logpush
-version: "0.5.1"
+version: "0.6.0"
 license: basic
 description: Collect and parse logs from Cloudflare API with Elastic Agent.
 type: integration