Merge branch 'main' of github.com:milan-elastic/integrations into mon…

…godb-atlas-database-logs

.buildkite/pipeline.publish.yml

@@ -37,7 +37,8 @@ steps:
       memory: "8G"
     env:
       ARTIFACTS_FOLDER: "artifacts-to-sign"
-      DRY_RUN: "true"
+      # by default it will publish packages
+      DRY_RUN: "${DRY_RUN:-false}"
     depends_on:
       - step: "check"
         allow_failure: false

.buildkite/pipeline.yml

@@ -1,5 +1,4 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
-
 env:
   SETUP_GVM_VERSION: "v0.5.2"
   LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"
@@ -29,22 +28,6 @@ steps:
       cpu: "8"
       memory: "4G"
 
-  - label: ":package: Build packages"
-    key: "build-packages"
-    command: ".buildkite/scripts/build_packages.sh"
-    agents:
-      image: "${LINUX_AGENT_IMAGE}"
-      cpu: "8"
-      memory: "8G"
-    env:
-      ARTIFACTS_FOLDER: "artifacts-to-sign"
-      DRY_RUN: "false"
-    depends_on:
-      - step: "check"
-        allow_failure: false
-    artifact_paths:
-      - artifacts-to-sign/*.zip
-
   - label: "Trigger integrations"
     key: "test-integrations"
     command: ".buildkite/scripts/trigger_integrations_in_parallel.sh"

.buildkite/scripts/common.sh

@@ -309,7 +309,6 @@ create_kind_cluster() {
     kind create cluster --config "${WORKSPACE}/kind-config.yaml" --image "kindest/node:${K8S_VERSION}"
 }
 
-
 delete_kind_cluster() {
     echo "--- Delete kind cluster"
     kind delete cluster || true
@@ -410,7 +409,6 @@ is_package_excluded() {
     return 1
 }
 
-
 is_supported_capability() {
     if [ "${SERVERLESS_PROJECT}" == "" ]; then
         return 0
@@ -755,6 +753,19 @@ build_zip_package() {
     return 0
 }
 
+skip_installation_step() {
+    local package=$1
+    if ! is_serverless ; then
+        return 1
+    fi
+
+    if [[ "$package" == "security_detection_engine" ]]; then
+        return 0
+    fi
+
+    return 1
+}
+
 install_package() {
     local package=$1
     echo "Install package: ${package}"
@@ -814,10 +825,13 @@ run_tests_package() {
         fi
     fi
 
-    echo "--- [${package}] test installation"
-    if ! install_package "${package}" ; then
-        return 1
+    if ! skip_installation_step "${package}" ; then
+        echo "--- [${package}] test installation"
+        if ! install_package "${package}" ; then
+            return 1
+        fi
     fi
+
     echo "--- [${package}] run test suites"
     if is_serverless; then
         if ! test_package_in_serverless "${package}" ; then
@@ -877,6 +891,10 @@ upload_safe_logs_from_package() {
     fi
 
     local package=$1
+    local retry_count="${BUILDKITE_RETRY_COUNT:-"0"}"
+    if [[ "${retry_count}" -ne 0 ]]; then
+        package="${package}_retry_${retry_count}"
+    fi
     local build_directory=$2
 
     local parent_folder="insecure-logs"

.github/CODEOWNERS

@@ -97,6 +97,7 @@
 /packages/azure_metrics/data_stream/storage_account @elastic/obs-ds-hosted-services
 /packages/barracuda @elastic/security-service-integrations
 /packages/barracuda_cloudgen_firewall @elastic/security-service-integrations
+/packages/bbot @elastic/security-service-integrations
 /packages/beaconing @elastic/ml-ui @elastic/sec-applied-ml
 /packages/beat @elastic/stack-monitoring
 /packages/bitdefender @elastic/security-service-integrations
@@ -146,6 +147,7 @@
 /packages/elastic_package_registry @elastic/ecosystem
 /packages/elasticsearch @elastic/stack-monitoring
 /packages/enterprisesearch @elastic/stack-monitoring
+/packages/entityanalytics_ad @elastic/security-service-integrations
 /packages/entityanalytics_entra_id @elastic/security-service-integrations
 /packages/entityanalytics_okta @elastic/security-service-integrations
 /packages/eset_protect @elastic/security-service-integrations
@@ -302,7 +304,6 @@
 /packages/system/changelog.yml @elastic/obs-infraobs-integrations @elastic/sec-linux-platform @elastic/sec-windows-platform
 /packages/system/data_stream/auth @elastic/sec-windows-platform
 /packages/system/data_stream/security @elastic/sec-linux-platform @elastic/sec-windows-platform
-/packages/system/kibana @elastic/elastic-agent-data-plane @elastic/kibana-visualizations
 /packages/system/manifest.yml @elastic/obs-infraobs-integrations @elastic/sec-linux-platform @elastic/sec-windows-platform
 /packages/system_audit @elastic/sec-linux-platform
 /packages/tanium @elastic/security-service-integrations

catalog-info.yaml

@@ -50,7 +50,7 @@ spec:
         build_tags: false
         filter_enabled: true
         filter_condition: >-
-          build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null)
+          build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null && build.source == 'api')
       repository: elastic/integrations
       cancel_intermediate_builds: true
       cancel_intermediate_builds_branch_filter: '!main !backport-*'
@@ -99,7 +99,7 @@ spec:
         build_tags: false
         filter_enabled: true
         filter_condition: >-
-          build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null)
+          build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null && build.source == 'api')
       repository: elastic/integrations
       cancel_intermediate_builds: true
       cancel_intermediate_builds_branch_filter: '!main'
@@ -146,7 +146,7 @@ spec:
         build_tags: false
         filter_enabled: true
         filter_condition: >-
-          build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null)
+          build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null && build.source == 'api')
       repository: elastic/integrations
       cancel_intermediate_builds: true
       cancel_intermediate_builds_branch_filter: '!main'
@@ -188,7 +188,7 @@ spec:
         build_tags: false
         filter_enabled: true
         filter_condition: >-
-          build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null)
+          build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null && build.source == 'api')
       repository: elastic/integrations
       cancel_intermediate_builds: true
       cancel_intermediate_builds_branch_filter: '!main'
@@ -232,7 +232,7 @@ spec:
         build_tags: false
         filter_enabled: true
         filter_condition: >-
-          build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null)
+          build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null && build.source == 'api')
       repository: elastic/integrations
       cancel_intermediate_builds: true
       cancel_intermediate_builds_branch_filter: '!main !backport-*'

packages/apache/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.17.1"
+  changes:
+    - description: Update grok for accepting user-identity.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/9632
 - version: "1.17.0"
   changes:
     - description: Limit request tracer log count to five.

packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log

@@ -7,4 +7,5 @@ monitoring-server - - [29/May/2017:19:02:48 +0000] "GET /status HTTP/1.1" 200 61
 monitoring-server - - [29/May/2017:19:02:48 +0000] "GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" X-Forwarded-For="-"
 89.160.20.112 - - [29/May/2017:19:02:48 +0000] "GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" X-Forwarded-For="10.0.0.2,10.0.0.1"
 2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6 - - [29/May/2017:19:02:48 +0000] "GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" X-Forwarded-For="10.225.192.17, 10.2.2.121"
-monitoring-server - - [17/May/2022:21:41:43 +0000] "GET / HTTP/1.1" 200 45 "-" "curl/7.79.1" X-Forwarded-For="192.168.0.2"
+monitoring-server - - [17/May/2022:21:41:43 +0000] "GET / HTTP/1.1" 200 45 "-" "curl/7.79.1" X-Forwarded-For="192.168.0.2"
+127.0.0.1 user-identity frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326

packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json

@@ -15,7 +15,7 @@
             "event": {
                 "category": "web",
                 "created": "2020-04-28T11:07:58.223Z",
-                "ingested": "2022-12-08T15:09:52.409634501Z",
+                "ingested": "2024-04-26T05:46:25.296250288Z",
                 "kind": "event",
                 "original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209",
                 "outcome": "failure"
@@ -63,7 +63,7 @@
             "event": {
                 "category": "web",
                 "created": "2020-04-28T11:07:58.223Z",
-                "ingested": "2022-12-08T15:09:52.409644668Z",
+                "ingested": "2024-04-26T05:46:25.296284705Z",
                 "kind": "event",
                 "original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"",
                 "outcome": "failure"
@@ -124,7 +124,7 @@
             "event": {
                 "category": "web",
                 "created": "2020-04-28T11:07:58.223Z",
-                "ingested": "2022-12-08T15:09:52.409645876Z",
+                "ingested": "2024-04-26T05:46:25.296289743Z",
                 "kind": "event",
                 "original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -",
                 "outcome": "failure"
@@ -160,7 +160,7 @@
             "event": {
                 "category": "web",
                 "created": "2020-04-28T11:07:58.223Z",
-                "ingested": "2022-12-08T15:09:52.409646876Z",
+                "ingested": "2024-04-26T05:46:25.296293311Z",
                 "kind": "event",
                 "original": "172.17.0.1 - - [29/May/2017:19:02:48 +0000] \"GET /stringpatch HTTP/1.1\" 404 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"",
                 "outcome": "failure"
@@ -221,7 +221,7 @@
             "event": {
                 "category": "web",
                 "created": "2020-04-28T11:07:58.223Z",
-                "ingested": "2022-12-08T15:09:52.409647793Z",
+                "ingested": "2024-04-26T05:46:25.296296691Z",
                 "kind": "event",
                 "original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /status HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"",
                 "outcome": "success"
@@ -282,7 +282,7 @@
             "event": {
                 "category": "web",
                 "created": "2020-04-28T11:07:58.223Z",
-                "ingested": "2022-12-08T15:09:52.409648793Z",
+                "ingested": "2024-04-26T05:46:25.296300048Z",
                 "kind": "event",
                 "original": "127.0.0.1 - - [02/Feb/2019:05:38:45 +0100] \"-\" 408 152 \"-\" \"-\"",
                 "outcome": "failure"
@@ -331,7 +331,7 @@
             "event": {
                 "category": "web",
                 "created": "2020-04-28T11:07:58.223Z",
-                "ingested": "2022-12-08T15:09:52.409649793Z",
+                "ingested": "2024-04-26T05:46:25.296303835Z",
                 "kind": "event",
                 "original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" X-Forwarded-For=\"-\"",
                 "outcome": "success"
@@ -398,7 +398,7 @@
             "event": {
                 "category": "web",
                 "created": "2020-04-28T11:07:58.223Z",
-                "ingested": "2022-12-08T15:09:52.409650668Z",
+                "ingested": "2024-04-26T05:46:25.296310193Z",
                 "kind": "event",
                 "original": "89.160.20.112 - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" X-Forwarded-For=\"10.0.0.2,10.0.0.1\"",
                 "outcome": "success"
@@ -486,7 +486,7 @@
             "event": {
                 "category": "web",
                 "created": "2020-04-28T11:07:58.223Z",
-                "ingested": "2022-12-08T15:09:52.409651543Z",
+                "ingested": "2024-04-26T05:46:25.296313609Z",
                 "kind": "event",
                 "original": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6 - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" X-Forwarded-For=\"10.225.192.17, 10.2.2.121\"",
                 "outcome": "success"
@@ -564,7 +564,7 @@
             "event": {
                 "category": "web",
                 "created": "2020-04-28T11:07:58.223Z",
-                "ingested": "2022-12-08T15:09:52.409652876Z",
+                "ingested": "2024-04-26T05:46:25.296316938Z",
                 "kind": "event",
                 "original": "monitoring-server - - [17/May/2022:21:41:43 +0000] \"GET / HTTP/1.1\" 200 45 \"-\" \"curl/7.79.1\" X-Forwarded-For=\"192.168.0.2\"",
                 "outcome": "success"
@@ -607,6 +607,55 @@
                 "original": "curl/7.79.1",
                 "version": "7.79.1"
             }
+        },
+        {
+            "@timestamp": "2000-10-10T20:55:36.000Z",
+            "apache": {
+                "access": {
+                    "identity": "user-identity",
+                    "remote_addresses": [
+                        "127.0.0.1"
+                    ]
+                }
+            },
+            "ecs": {
+                "version": "8.5.1"
+            },
+            "event": {
+                "category": "web",
+                "created": "2020-04-28T11:07:58.223Z",
+                "ingested": "2024-04-26T05:46:25.296320274Z",
+                "kind": "event",
+                "original": "127.0.0.1 user-identity frank [10/Oct/2000:13:55:36 -0700] \"GET /apache_pb.gif HTTP/1.0\" 200 2326",
+                "outcome": "success"
+            },
+            "http": {
+                "request": {
+                    "method": "GET"
+                },
+                "response": {
+                    "body": {
+                        "bytes": 2326
+                    },
+                    "status_code": 200
+                },
+                "version": "1.0"
+            },
+            "source": {
+                "address": "127.0.0.1",
+                "ip": "127.0.0.1"
+            },
+            "tags": [
+                "preserve_original_event"
+            ],
+            "url": {
+                "extension": "gif",
+                "original": "/apache_pb.gif",
+                "path": "/apache_pb.gif"
+            },
+            "user": {
+                "name": "frank"
+            }
         }
     ]
 }

packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json

@@ -15,7 +15,7 @@
             "event": {
                 "category": "web",
                 "created": "2020-04-28T11:07:58.223Z",
-                "ingested": "2022-12-08T15:09:52.483539043Z",
+                "ingested": "2024-04-26T05:46:25.447843628Z",
                 "kind": "event",
                 "original": "::1 - - [26/Dec/2016:16:16:28 +0200] \"GET / HTTP/1.1\" 200 45",
                 "outcome": "success"
@@ -62,7 +62,7 @@
             "event": {
                 "category": "web",
                 "created": "2020-04-28T11:07:58.223Z",
-                "ingested": "2022-12-08T15:09:52.483550209Z",
+                "ingested": "2024-04-26T05:46:25.447895323Z",
                 "kind": "event",
                 "original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209",
                 "outcome": "failure"
@@ -110,7 +110,7 @@
             "event": {
                 "category": "web",
                 "created": "2020-04-28T11:07:58.223Z",
-                "ingested": "2022-12-08T15:09:52.483551501Z",
+                "ingested": "2024-04-26T05:46:25.447905030Z",
                 "kind": "event",
                 "original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -",
                 "outcome": "failure"
@@ -146,7 +146,7 @@
             "event": {
                 "category": "web",
                 "created": "2020-04-28T11:07:58.223Z",
-                "ingested": "2022-12-08T15:09:52.483552501Z",
+                "ingested": "2024-04-26T05:46:25.447912585Z",
                 "kind": "event",
                 "original": "89.160.20.156 - - [26/Dec/2016:18:23:35 +0200] \"GET / HTTP/1.1\" 200 45",
                 "outcome": "success"
@@ -211,7 +211,7 @@
             "event": {
                 "category": "web",
                 "created": "2020-04-28T11:07:58.223Z",
-                "ingested": "2022-12-08T15:09:52.483553418Z",
+                "ingested": "2024-04-26T05:46:25.447919912Z",
                 "kind": "event",
                 "original": "89.160.20.156 - - [26/Dec/2016:18:23:41 +0200] \"GET /notfound HTTP/1.1\" 404 206",
                 "outcome": "failure"
@@ -276,7 +276,7 @@
             "event": {
                 "category": "web",
                 "created": "2020-04-28T11:07:58.223Z",
-                "ingested": "2022-12-08T15:09:52.483554501Z",
+                "ingested": "2024-04-26T05:46:25.447927217Z",
                 "kind": "event",
                 "original": "89.160.20.156 - - [26/Dec/2016:18:23:45 +0200] \"GET /hmm HTTP/1.1\" 404 201",
                 "outcome": "failure"