Merge branch 'main' into main

.buildkite/auditbeat/auditbeat-pipeline.yml

@@ -66,7 +66,7 @@ steps:
           - "auditbeat/build/*.xml"
           - "auditbeat/build/*.json"
 
-      - label: ":windows: Auditbeat Win-2016 Auditbeat Unit Tests"
+      - label: ":windows: Auditbeat Win-2016 Unit Tests"
         command: |
           Set-Location -Path $BEATS_PROJECT_NAME
           mage build unitTest
@@ -116,7 +116,7 @@ steps:
           instanceType: "${AWS_ARM_INSTANCE_TYPE}"
         artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
 
-  - group: "Heartbeat MacOS Extended"
+  - group: "Auditbeat MacOS Extended"
     if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
     steps:
       - label: ":mac: MacOS Unit Tests"
@@ -137,7 +137,7 @@ steps:
     key: "auditbeat-extended-tests-win"
     if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/
     steps:
-      - label: ":windows: Auditbeat Windows 2019 Unit Tests"
+      - label: ":windows: Auditbeat Win-2019 Unit Tests"
         key: "auditbeat-extended-win-2019-unit-tests"
         command: "mage -d ${BEATS_PROJECT_NAME} unitTest"
         agents:
@@ -148,7 +148,7 @@ steps:
           disk_type: "pd-ssd"
         artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
 
-      - label: ":windows: Auditbeat Windows 10 Unit Tests"
+      - label: ":windows: Auditbeat Win-10 Unit Tests"
         key: "auditbeat-extended-win-10-unit-tests"
         command: "mage -d ${BEATS_PROJECT_NAME} unitTest"
         agents:
@@ -159,7 +159,7 @@ steps:
           disk_type: "pd-ssd"
         artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
 
-      - label: ":windows: Windows 11 Unit Tests"
+      - label: ":windows: Auditbeat Win-11 Unit Tests"
         key: "auditbeat-extended-win-11-unit-tests"
         command: "mage -d ${BEATS_PROJECT_NAME} unitTest"
         agents:
@@ -175,11 +175,12 @@ steps:
     if: build.env("BUILDKITE_PULL_REQUEST") != "false"
     depends_on:
       - "auditbeat-mandatory-tests"
+
     steps:
       - label: ":ubuntu: Auditbeat/Packaging Linux X86"
         key: "auditbeat-package-linux-x86"
         env:
-          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
+          PLATFORMS: "${PACKAGING_PLATFORMS}"
           SNAPSHOT: true
         command: "cd $BEATS_PROJECT_NAME && mage package"
         agents:
@@ -190,7 +191,7 @@ steps:
       - label: ":linux: Auditbeat/Packaging Linux ARM"
         key: "auditbeat-package-linux-arm"
         env:
-          PLATFORMS: "linux/arm64"
+          PLATFORMS: "${PACKAGING_ARM_PLATFORMS}"
           PACKAGES: "docker"
           SNAPSHOT: true
         command: "cd $BEATS_PROJECT_NAME && mage package"

.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml

@@ -4,62 +4,81 @@ env:
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
   GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
 
+notify:
+  - github_commit_status:
+      context: "Deploy/K8S"
+
 steps:
-  - label: "Checks"
-    command: ".buildkite/deploy/kubernetes/scripts/make.sh"
-    if: build.env("BUILDKITE_PULL_REQUEST") != "false" && build.env("GITHUB_PR_LABELS") == "kubernetes"
-    notify:
-      - github_commit_status:
-          context: "Deploy K8S/Checks"
-    agents:
-      provider: "gcp"
-      image: "${IMAGE_UBUNTU_X86_64}"
-      machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+  - group: "Depoy/K8S"
+    if: build.env("BUILDKITE_PULL_REQUEST") != "false" && build.env("GITHUB_PR_LABELS") =~ /.*kubernetes.*/
+
+    steps:
+    - label: "Checks"
+      command: ".buildkite/deploy/kubernetes/scripts/make.sh"
+      notify:
+        - github_commit_status:
+            context: "Deploy/k8s-checks"
+      agents:
+        provider: "gcp"
+        image: "${IMAGE_UBUNTU_X86_64}"
+        machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
 
-  - label: "K8S Test/K8S version: v1.29.0"
-    key: "k8s-test-129"
-    env:
-      K8S_VERSION: "v1.29.0"
-    commands:
-      - "MODULE=kubernetes make -C metricbeat integration-tests"
-      - "make -C deploy/kubernetes test"
-    agents:
-      provider: "gcp"
-      image: "${IMAGE_UBUNTU_X86_64}"
-      machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+    - label: "K8S Test/K8S version: v1.29.0"
+      key: "k8s-test-129"
+      env:
+        K8S_VERSION: "v1.29.0"
+      commands:
+        - "MODULE=kubernetes make -C metricbeat integration-tests"
+        - "make -C deploy/kubernetes test"
+      notify:
+        - github_commit_status:
+            context: "Deploy/k8s-test v1.29.0"
+      agents:
+        provider: "gcp"
+        image: "${IMAGE_UBUNTU_X86_64}"
+        machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
 
-  - label: "K8S Test/K8S version: v1.28.0"
-    key: "k8s-test-128"
-    env:
-      K8S_VERSION: "v1.28.0"
-    commands:
-      - "MODULE=kubernetes make -C metricbeat integration-tests"
-      - "make -C deploy/kubernetes test"
-    agents:
-      provider: "gcp"
-      image: "${IMAGE_UBUNTU_X86_64}"
-      machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+    - label: "K8S Test/K8S version: v1.28.0"
+      key: "k8s-test-128"
+      env:
+        K8S_VERSION: "v1.28.0"
+      commands:
+        - "MODULE=kubernetes make -C metricbeat integration-tests"
+        - "make -C deploy/kubernetes test"
+      notify:
+        - github_commit_status:
+            context: "Deploy/k8s-test v1.28.0"
+      agents:
+        provider: "gcp"
+        image: "${IMAGE_UBUNTU_X86_64}"
+        machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
 
-  - label: "K8S Test/K8S version: v1.27.3"
-    key: "k8s-test-1273"
-    env:
-      K8S_VERSION: "v1.27.3"
-    commands:
-      - "MODULE=kubernetes make -C metricbeat integration-tests"
-      - "make -C deploy/kubernetes test"
-    agents:
-      provider: "gcp"
-      image: "${IMAGE_UBUNTU_X86_64}"
-      machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+    - label: "K8S Test/K8S version: v1.27.3"
+      key: "k8s-test-1273"
+      env:
+        K8S_VERSION: "v1.27.3"
+      commands:
+        - "MODULE=kubernetes make -C metricbeat integration-tests"
+        - "make -C deploy/kubernetes test"
+      notify:
+        - github_commit_status:
+            context: "Deploy/k8s-test v1.27.3"
+      agents:
+        provider: "gcp"
+        image: "${IMAGE_UBUNTU_X86_64}"
+        machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
 
-  - label: "K8S Test/K8S version: v1.26.6"
-    key: "k8s-test-1266"
-    env:
-      K8S_VERSION: "v1.26.6"
-    commands:
-      - "MODULE=kubernetes make -C metricbeat integration-tests"
-      - "make -C deploy/kubernetes test"
-    agents:
-      provider: "gcp"
-      image: "${IMAGE_UBUNTU_X86_64}"
-      machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+    - label: "K8S Test/K8S version: v1.26.6"
+      key: "k8s-test-1266"
+      env:
+        K8S_VERSION: "v1.26.6"
+      commands:
+        - "MODULE=kubernetes make -C metricbeat integration-tests"
+        - "make -C deploy/kubernetes test"
+      notify:
+        - github_commit_status:
+            context: "Deploy/k8s-test v1.26.6"
+      agents:
+        provider: "gcp"
+        image: "${IMAGE_UBUNTU_X86_64}"
+        machineType: "${GCP_HI_PERF_MACHINE_TYPE}"

.buildkite/heartbeat/heartbeat-pipeline.yml

@@ -5,21 +5,26 @@ env:
 
   ASDF_MAGE_VERSION: 1.15.0
 
-  IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
+  AWS_ARM_INSTANCE_TYPE: "m6g.xlarge"
   AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
+
+  GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
+  GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
+  GCP_WIN_MACHINE_TYPE: "n2-standard-8"
+
+  IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
+  IMAGE_MACOS_ARM: "generic-13-ventura-arm"
   IMAGE_RHEL9: "family/platform-ingest-beats-rhel-9"
+  IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
   IMAGE_WIN_10: "family/platform-ingest-beats-windows-10"
   IMAGE_WIN_11: "family/platform-ingest-beats-windows-11"
   IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016"
   IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019"
   IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022"
-  IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
-  IMAGE_MACOS_ARM: "generic-13-ventura-arm"
 
-  GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
-  GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
-  GCP_WIN_MACHINE_TYPE: "n2-standard-8"
-  AWS_ARM_INSTANCE_TYPE: "m6g.xlarge"
+  #Packaging
+  PACKAGING_ARM_PLATFORMS: "linux/arm64"
+  PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
 
 notify:
   - github_commit_status:
@@ -178,7 +183,7 @@ steps:
       - label: ":ubuntu: Packaging Linux X86"
         key: "heartbeat-package-linux-x86"
         env:
-          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
+          PLATFORMS: "${PACKAGING_PLATFORMS}"
           SNAPSHOT: true
         command: "cd $BEATS_PROJECT_NAME && mage package"
         agents:
@@ -189,7 +194,7 @@ steps:
       - label: ":linux: Packaging Linux ARM"
         key: "heartbeat-package-linux-arm"
         env:
-          PLATFORMS: "linux/arm64"
+          PLATFORMS: "${PACKAGING_ARM_PLATFORMS}"
           PACKAGES: "docker"
           SNAPSHOT: true
         command: "cd $BEATS_PROJECT_NAME && mage package"

.buildkite/hooks/pre-command

@@ -11,7 +11,7 @@ PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-inges
 DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod"
 GITHUB_TOKEN_VAULT_PATH="kv/ci-shared/platform-ingest/github_token"
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" || "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" || "$BUILDKITE_PIPELINE_SLUG" == "deploy-k8s" ]]; then
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" || "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" ]]; then
   source .buildkite/env-scripts/env.sh
   if [[ -z "${GO_VERSION-""}" ]]; then
     export GO_VERSION=$(cat "${WORKSPACE}/.go-version")
@@ -35,13 +35,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" || "$BUILDKITE_PIPELINE_SLUG" == "fi
     with_mage
   fi
 
-  if [[ "$BUILDKITE_STEP_KEY" == windows* ]]; then
-    git config core.autocrlf false
-    git rm --quiet --cached -r .
-    git reset --quiet --hard
-  fi
-
-  if [[ "$BUILDKITE_STEP_KEY" == package* ]]; then
+  if [[ "$BUILDKITE_STEP_KEY" == *packaging* ]]; then
     DOCKER_USERNAME_SECRET=$(retry_with_count 5 vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}")
     DOCKER_PASSWORD_SECRET=$(retry_with_count 5 vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}")
     GITHUB_TOKEN_SECRET=$(retry_with_count 5 vault kv get -field token ${GITHUB_TOKEN_VAULT_PATH})
@@ -58,17 +52,11 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" || "$BUILDKITE_PIPELINE_SLUG" == "fi
     git config user.name "$github_username"
     git config user.email "$github_email"
   fi
-
-  if [[ "$BUILDKITE_STEP_KEY" == k8s-test* ]]; then
-    .buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh
-  fi
 fi
 
 ENABLED_BEATS_PIPELINES_SLUGS=(
-  "beats"
   "auditbeat"
   "filebeat"
-  "beats-metricbeat"
   "beats-libbeat"
   "beats-packetbeat"
   "beats-winlogbeat"
@@ -82,7 +70,8 @@ ENABLED_BEATS_PIPELINES_SLUGS=(
   "beats-xpack-filebeat"
   "beats-xpack-heartbeat"
   "beats-xpack-osquerybeat"
-  )
+  "deploy-k8s"
+)
 
 for slug in "${ENABLED_BEATS_PIPELINES_SLUGS[@]}"; do
   if [[ "$BUILDKITE_PIPELINE_SLUG" == "$slug" ]]; then
@@ -106,3 +95,10 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" ]]; then
     export PRIVATE_CI_GCS_CREDENTIALS_SECRET
   fi
 fi
+
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "deploy-k8s" ]]; then
+  source .buildkite/env-scripts/env.sh
+  if [[ "$BUILDKITE_STEP_KEY" == k8s-test* ]]; then
+    .buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh
+  fi
+fi