Merge branch 'main' of github.com:niraj-elastic/integrations into pac…

…kage_mongodb_atlas_hardware

.buildkite/hooks/pre-command

@@ -82,12 +82,6 @@ if [[ "${BUILDKITE_PIPELINE_SLUG}" == "integrations-publish" ]]; then
 fi
 
 if [[ "${BUILDKITE_PIPELINE_SLUG}" == "integrations" ]]; then
-    if [[ "${BUILDKITE_STEP_KEY}" == "trigger-publish" ]]; then
-        # TODO: To be removed
-        BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field buildkite_token ${BUILDKITE_API_TOKEN_PATH})
-        export BUILDKITE_API_TOKEN
-    fi
-
     if [[ "${BUILDKITE_STEP_KEY}" == "test-integrations" ]]; then
         BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field buildkite_token "${BUILDKITE_API_TOKEN_PATH}")
         export BUILDKITE_API_TOKEN

.buildkite/pipeline.publish.yml

@@ -9,8 +9,7 @@ env:
   JQ_VERSION: '1.7'
   # Elastic package settings
   # Manage docker output/logs
-  ELASTIC_PACKAGE_COMPOSE_DISABLE_ANSI: "true"
-  ELASTIC_PACKAGE_COMPOSE_DISABLE_PULL_PROGRESS_INFORMATION: "true"
+  ELASTIC_PACKAGE_COMPOSE_DISABLE_VERBOSE_OUTPUT: "true"
   # Default license to use by `elastic-package build`
   ELASTIC_PACKAGE_REPOSITORY_LICENSE: "licenses/Elastic-2.0.txt"
   # Link definitions path (full path to be set in the corresponding step)
@@ -37,7 +36,8 @@ steps:
       memory: "8G"
     env:
       ARTIFACTS_FOLDER: "artifacts-to-sign"
-      DRY_RUN: "true"
+      # by default it will publish packages
+      DRY_RUN: "${DRY_RUN:-false}"
     depends_on:
       - step: "check"
         allow_failure: false

.buildkite/pipeline.yml

@@ -1,10 +1,9 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
-
 env:
   SETUP_GVM_VERSION: "v0.5.2"
   LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"
   DOCKER_COMPOSE_VERSION: "v2.24.1"
-  DOCKER_VERSION: "false"
+  DOCKER_VERSION: "26.1.0"
   KIND_VERSION: 'v0.20.0'
   K8S_VERSION: 'v1.29.0'
   YQ_VERSION: 'v4.35.2'
@@ -29,22 +28,6 @@ steps:
       cpu: "8"
       memory: "4G"
 
-  - label: ":package: Build packages"
-    key: "build-packages"
-    command: ".buildkite/scripts/build_packages.sh"
-    agents:
-      image: "${LINUX_AGENT_IMAGE}"
-      cpu: "8"
-      memory: "8G"
-    env:
-      ARTIFACTS_FOLDER: "artifacts-to-sign"
-      DRY_RUN: "false"
-    depends_on:
-      - step: "check"
-        allow_failure: false
-    artifact_paths:
-      - artifacts-to-sign/*.zip
-
   - label: "Trigger integrations"
     key: "test-integrations"
     command: ".buildkite/scripts/trigger_integrations_in_parallel.sh"

.buildkite/scripts/common.sh

@@ -162,9 +162,14 @@ with_docker() {
     fi
     echo "deb [arch=${architecture} signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu ${ubuntu_codename} stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
     sudo apt-get update
-    sudo DEBIAN_FRONTEND=noninteractive apt-get install --allow-downgrades -y "docker-ce=${debian_version}"
-    sudo DEBIAN_FRONTEND=noninteractive apt-get install --allow-downgrades -y "docker-ce-cli=${debian_version}"
+    sudo DEBIAN_FRONTEND=noninteractive apt-get install --allow-change-held-packages --allow-downgrades -y "docker-ce=${debian_version}"
+    sudo DEBIAN_FRONTEND=noninteractive apt-get install --allow-change-held-packages --allow-downgrades -y "docker-ce-cli=${debian_version}"
     sudo systemctl start docker
+
+    echo "- Installed docker client version:"
+    docker version -f json  | jq -r '.Client.Version'
+    echo "- Installed docker server version:"
+    docker version -f json  | jq -r '.Server.Version'
 }
 
 with_docker_compose_plugin() {
@@ -309,7 +314,6 @@ create_kind_cluster() {
     kind create cluster --config "${WORKSPACE}/kind-config.yaml" --image "kindest/node:${K8S_VERSION}"
 }
 
-
 delete_kind_cluster() {
     echo "--- Delete kind cluster"
     kind delete cluster || true
@@ -410,7 +414,6 @@ is_package_excluded() {
     return 1
 }
 
-
 is_supported_capability() {
     if [ "${SERVERLESS_PROJECT}" == "" ]; then
         return 0
@@ -755,6 +758,19 @@ build_zip_package() {
     return 0
 }
 
+skip_installation_step() {
+    local package=$1
+    if ! is_serverless ; then
+        return 1
+    fi
+
+    if [[ "$package" == "security_detection_engine" ]]; then
+        return 0
+    fi
+
+    return 1
+}
+
 install_package() {
     local package=$1
     echo "Install package: ${package}"
@@ -814,10 +830,13 @@ run_tests_package() {
         fi
     fi
 
-    echo "--- [${package}] test installation"
-    if ! install_package "${package}" ; then
-        return 1
+    if ! skip_installation_step "${package}" ; then
+        echo "--- [${package}] test installation"
+        if ! install_package "${package}" ; then
+            return 1
+        fi
     fi
+
     echo "--- [${package}] run test suites"
     if is_serverless; then
         if ! test_package_in_serverless "${package}" ; then
@@ -877,6 +896,10 @@ upload_safe_logs_from_package() {
     fi
 
     local package=$1
+    local retry_count="${BUILDKITE_RETRY_COUNT:-"0"}"
+    if [[ "${retry_count}" -ne 0 ]]; then
+        package="${package}_retry_${retry_count}"
+    fi
     local build_directory=$2
 
     local parent_folder="insecure-logs"

.github/CODEOWNERS

@@ -60,6 +60,7 @@
 /packages/aws/data_stream/vpn @elastic/obs-infraobs-integrations
 /packages/aws/kibana @elastic/obs-ds-hosted-services @elastic/kibana-visualizations @elastic/obs-infraobs-integrations
 /packages/aws/manifest.yml @elastic/obs-ds-hosted-services @elastic/security-service-integrations @elastic/obs-infraobs-integrations
+/packages/aws_bedrock @elastic/security-service-integrations
 /packages/aws_logs @elastic/obs-ds-hosted-services
 /packages/awsfargate @elastic/obs-infraobs-integrations
 /packages/awsfirehose @elastic/obs-ds-hosted-services
@@ -97,6 +98,7 @@
 /packages/azure_metrics/data_stream/storage_account @elastic/obs-ds-hosted-services
 /packages/barracuda @elastic/security-service-integrations
 /packages/barracuda_cloudgen_firewall @elastic/security-service-integrations
+/packages/bbot @elastic/security-service-integrations
 /packages/beaconing @elastic/ml-ui @elastic/sec-applied-ml
 /packages/beat @elastic/stack-monitoring
 /packages/bitdefender @elastic/security-service-integrations
@@ -137,6 +139,7 @@
 /packages/crowdstrike @elastic/security-service-integrations
 /packages/cyberark_pta @elastic/security-service-integrations
 /packages/cyberarkpas @elastic/security-service-integrations
+/packages/cybereason @elastic/security-service-integrations
 /packages/cylance @elastic/security-service-integrations
 /packages/darktrace @elastic/security-service-integrations
 /packages/ded @elastic/ml-ui @elastic/sec-applied-ml
@@ -146,6 +149,7 @@
 /packages/elastic_package_registry @elastic/ecosystem
 /packages/elasticsearch @elastic/stack-monitoring
 /packages/enterprisesearch @elastic/stack-monitoring
+/packages/entityanalytics_ad @elastic/security-service-integrations
 /packages/entityanalytics_entra_id @elastic/security-service-integrations
 /packages/entityanalytics_okta @elastic/security-service-integrations
 /packages/eset_protect @elastic/security-service-integrations
@@ -295,14 +299,14 @@
 /packages/suricata @elastic/sec-deployment-and-devices
 /packages/symantec_edr_cloud @elastic/security-service-integrations
 /packages/symantec_endpoint @elastic/security-service-integrations
+/packages/symantec_endpoint_security @elastic/security-service-integrations
 /packages/synthetics @elastic/obs-ux-infra_services-team
 /packages/synthetics_dashboards @elastic/obs-ux-infra_services-team
 /packages/sysmon_linux @elastic/sec-linux-platform
 /packages/system @elastic/obs-infraobs-integrations
 /packages/system/changelog.yml @elastic/obs-infraobs-integrations @elastic/sec-linux-platform @elastic/sec-windows-platform
 /packages/system/data_stream/auth @elastic/sec-windows-platform
 /packages/system/data_stream/security @elastic/sec-linux-platform @elastic/sec-windows-platform
-/packages/system/kibana @elastic/elastic-agent-data-plane @elastic/kibana-visualizations
 /packages/system/manifest.yml @elastic/obs-infraobs-integrations @elastic/sec-linux-platform @elastic/sec-windows-platform
 /packages/system_audit @elastic/sec-linux-platform
 /packages/tanium @elastic/security-service-integrations

catalog-info.yaml

@@ -50,7 +50,7 @@ spec:
         build_tags: false
         filter_enabled: true
         filter_condition: >-
-          build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null)
+          build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null && build.source == 'api')
       repository: elastic/integrations
       cancel_intermediate_builds: true
       cancel_intermediate_builds_branch_filter: '!main !backport-*'
@@ -99,7 +99,7 @@ spec:
         build_tags: false
         filter_enabled: true
         filter_condition: >-
-          build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null)
+          build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null && build.source == 'api')
       repository: elastic/integrations
       cancel_intermediate_builds: true
       cancel_intermediate_builds_branch_filter: '!main'
@@ -146,7 +146,7 @@ spec:
         build_tags: false
         filter_enabled: true
         filter_condition: >-
-          build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null)
+          build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null && build.source == 'api')
       repository: elastic/integrations
       cancel_intermediate_builds: true
       cancel_intermediate_builds_branch_filter: '!main'
@@ -188,7 +188,7 @@ spec:
         build_tags: false
         filter_enabled: true
         filter_condition: >-
-          build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null)
+          build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null && build.source == 'api')
       repository: elastic/integrations
       cancel_intermediate_builds: true
       cancel_intermediate_builds_branch_filter: '!main'
@@ -222,17 +222,15 @@ spec:
       name: integrations-publish
       description: 'Pipeline for the Integrations project to publish packages'
     spec:
-      # TODO: add backport-* in `branch_configuration` when these brances are updated
-      # branch_configuration: "main backport-*"
-      branch_configuration: "main"
+      branch_configuration: "main backport-*"
       pipeline_file: ".buildkite/pipeline.publish.yml"
       provider_settings:
         build_pull_request_forks: false
         build_pull_requests: false # requires filter_enabled and filter_condition settings as below when used with buildkite-pr-bot
         build_tags: false
         filter_enabled: true
         filter_condition: >-
-          build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null)
+          build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null && build.source == 'api')
       repository: elastic/integrations
       cancel_intermediate_builds: true
       cancel_intermediate_builds_branch_filter: '!main !backport-*'

go.mod

@@ -4,7 +4,7 @@ go 1.22.0
 
 require (
 	github.com/blang/semver v3.5.1+incompatible
-	github.com/elastic/elastic-package v0.99.0
+	github.com/elastic/elastic-package v0.100.0
 	github.com/elastic/go-licenser v0.4.1
 	github.com/elastic/package-registry v1.24.0
 	github.com/magefile/mage v1.15.0
@@ -30,7 +30,7 @@ require (
 	github.com/PaesslerAG/gval v1.2.1 // indirect
 	github.com/PaesslerAG/jsonpath v0.1.1 // indirect
 	github.com/Pallinder/go-randomdata v1.2.0 // indirect
-	github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 // indirect
+	github.com/ProtonMail/go-crypto v1.0.0 // indirect
 	github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect
 	github.com/ProtonMail/gopenpgp/v2 v2.7.5 // indirect
 	github.com/andybalholm/brotli v1.0.4 // indirect
@@ -39,22 +39,23 @@ require (
 	github.com/aymerick/raymond v2.0.2+incompatible // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cbroglie/mustache v1.4.0 // indirect
-	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/chai2010/gettext-go v1.0.2 // indirect
 	github.com/cloudflare/circl v1.3.7 // indirect
+	github.com/creack/pty v1.1.21 // indirect
 	github.com/creasty/defaults v1.7.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/elastic/elastic-integration-corpus-generator-tool v0.10.0 // indirect
 	github.com/elastic/go-elasticsearch/v7 v7.17.10 // indirect
-	github.com/elastic/go-resource v0.1.1 // indirect
+	github.com/elastic/go-resource v0.2.0 // indirect
 	github.com/elastic/go-sysinfo v1.9.0 // indirect
 	github.com/elastic/go-ucfg v0.8.8 // indirect
 	github.com/elastic/go-windows v1.0.1 // indirect
 	github.com/elastic/gojsonschema v1.2.1 // indirect
 	github.com/elastic/kbncontent v0.1.3 // indirect
-	github.com/elastic/package-spec/v3 v3.1.3 // indirect
+	github.com/elastic/package-spec/v3 v3.1.4 // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/evanphx/json-patch v5.7.0+incompatible // indirect
 	github.com/evanphx/json-patch/v5 v5.9.0 // indirect
@@ -139,7 +140,7 @@ require (
 	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/rivo/uniseg v0.4.3 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	github.com/shirou/gopsutil/v3 v3.24.2 // indirect
+	github.com/shirou/gopsutil/v3 v3.24.4 // indirect
 	github.com/shoenig/go-m1cpu v0.1.6 // indirect
 	github.com/shopspring/decimal v1.3.1 // indirect
 	github.com/spf13/afero v1.11.0 // indirect
@@ -189,16 +190,16 @@ require (
 	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	helm.sh/helm/v3 v3.14.3 // indirect
+	helm.sh/helm/v3 v3.14.4 // indirect
 	howett.net/plist v1.0.0 // indirect
-	k8s.io/api v0.29.3 // indirect
+	k8s.io/api v0.30.0 // indirect
 	k8s.io/apiextensions-apiserver v0.29.0 // indirect
-	k8s.io/apimachinery v0.29.3 // indirect
-	k8s.io/cli-runtime v0.29.3 // indirect
-	k8s.io/client-go v0.29.3 // indirect
+	k8s.io/apimachinery v0.30.0 // indirect
+	k8s.io/cli-runtime v0.30.0 // indirect
+	k8s.io/client-go v0.30.0 // indirect
 	k8s.io/component-base v0.29.0 // indirect
-	k8s.io/klog/v2 v2.110.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect
+	k8s.io/klog/v2 v2.120.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
 	k8s.io/kubectl v0.29.0 // indirect
 	k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect