Merge branch 'main' into agentbeat

.buildkite/auditbeat/auditbeat-pipeline.yml

@@ -2,12 +2,17 @@
 name: "beats-auditbeat"
 
 env:
+  BEATS_PROJECT_NAME: "auditbeat"
+
+  ASDF_MAGE_VERSION: 1.15.0
+
   AWS_ARM_INSTANCE_TYPE: "m6g.xlarge"
   AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
-  BEATS_PROJECT_NAME: "auditbeat"
+
   GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
   GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
   GCP_WIN_MACHINE_TYPE: "n2-standard-8"
+
   IMAGE_MACOS_ARM: "generic-13-ventura-arm"
   IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
   IMAGE_RHEL9: "family/platform-ingest-beats-rhel-9"
@@ -22,56 +27,175 @@ env:
   PACKAGING_ARM_PLATFORMS: "linux/arm64"
   PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
 
-  #Deps
-  ASDF_MAGE_VERSION: 1.15.0
+notify:
+  - github_commit_status:
+      context: "$BEATS_PROJECT_NAME"
 
 steps:
-  - input: "Input Parameters"
-    key: "run_auditbeat"
-    fields:
-      - select: "auditbeat - run_auditbeat"
-        key: "run_auditbeat"
-        options:
-          - label: "True"
-            value: "true"
-          - label: "False"
-            value: "false"
-        default: "false"
-      - select: "auditbeat - run_auditbeat_macos_tests"
-        key: "run_auditbeat_macos_tests"
-        options:
-          - label: "True"
-            value: "true"
-          - label: "False"
-            value: "false"
-        default: "false"
-      - select: "auditbeat - run_auditbeat_arm_tests"
-        key: "run_auditbeat_arm_tests"
-        options:
-          - label: "True"
-            value: "true"
-          - label: "False"
-            value: "false"
-        default: "false"
-      - select: "auditbeat - run_auditbeat_win_tests"
-        key: "run_auditbeat_win_tests"
-        options:
-          - label: "True"
-            value: "true"
-          - label: "False"
-            value: "false"
-        default: "false"
-    if: "build.source == 'ui'"
-
-  - wait: ~
-    if: "build.source == 'ui'"
-    allow_dependency_failure: false
-
-  - label: ":linux: Load dynamic auditbeat pipeline"
-    key: "auditbeat-pipeline"
-    command: ".buildkite/scripts/generate_auditbeat_pipeline.sh"
-    agents:
-      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest"
-    notify:
-      - github_commit_status:
-          context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps"
+  - group: "Auditbeat Mandatory Testing"
+    key: "auditbeat-mandatory-tests"
+
+    steps:
+      - label: ":ubuntu: Auditbeat Unit Tests"
+        command: "cd $BEATS_PROJECT_NAME && mage build unitTest"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
+        artifact_paths:
+          - "auditbeat/build/*.xml"
+          - "auditbeat/build/*.json"
+
+      - label: ":ubuntu: Auditbeat Unit Tests"
+        command: "cd $BEATS_PROJECT_NAME && mage build unitTest"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
+        artifact_paths:
+          - "auditbeat/build/*.xml"
+          - "auditbeat/build/*.json"
+
+      - label: ":rhel: Auditbeat Unit Tests"
+        command: "cd $BEATS_PROJECT_NAME && mage build unitTest"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_RHEL9}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
+        artifact_paths:
+          - "auditbeat/build/*.xml"
+          - "auditbeat/build/*.json"
+
+      - label: ":windows: Auditbeat Win-2016 Unit Tests"
+        command: |
+          Set-Location -Path $BEATS_PROJECT_NAME
+          mage build unitTest
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_2016}"
+          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+          disk_size: 200
+          disk_type: "pd-ssd"
+        artifact_paths:
+          - "auditbeat/build/*.xml"
+          - "auditbeat/build/*.json"
+
+      - label: ":windows: Auditbeat Win-2022 Unit Tests"
+        command: |
+          Set-Location -Path $BEATS_PROJECT_NAME
+          mage build unitTest
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_2022}"
+          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+          disk_size: 200
+          disk_type: "pd-ssd"
+        artifact_paths:
+          - "auditbeat/build/*.xml"
+          - "auditbeat/build/*.json"
+
+      - label: ":linux: Auditbeat Crosscompile"
+        command: "make -C $BEATS_PROJECT_NAME crosscompile"
+        env:
+          GOX_FLAGS: "-arch amd64"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+
+  - group: "Auditbeat ARM Tests"
+    key: "auditbeat-extended-tests-arm"
+    if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
+    steps:
+      - label: ":arm: ARM64 Unit Tests"
+        key: "auditbeat-extended-arm64-unit-tests"
+        command: "cd ${BEATS_PROJECT_NAME} && mage build unitTest"
+        agents:
+          provider: "aws"
+          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
+          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+  - group: "Auditbeat MacOS Extended"
+    if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
+    steps:
+      - label: ":mac: MacOS Unit Tests"
+        command: "cd ${BEATS_PROJECT_NAME} && mage unitTest"
+        agents:
+          provider: "orka"
+          imagePrefix: "${IMAGE_MACOS_X86_64}"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+      - label: ":mac: MacOS ARM Unit Tests"
+        command: "cd ${BEATS_PROJECT_NAME} && mage unitTest"
+        agents:
+          provider: "orka"
+          imagePrefix: "${IMAGE_MACOS_ARM}"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+  - group: "Auditbeat Windows Extended Testing"
+    key: "auditbeat-extended-tests-win"
+    if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/
+    steps:
+      - label: ":windows: Auditbeat Win-2019 Unit Tests"
+        key: "auditbeat-extended-win-2019-unit-tests"
+        command: "mage -d ${BEATS_PROJECT_NAME} unitTest"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_2019}"
+          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+          disk_size: 100
+          disk_type: "pd-ssd"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+      - label: ":windows: Auditbeat Win-10 Unit Tests"
+        key: "auditbeat-extended-win-10-unit-tests"
+        command: "mage -d ${BEATS_PROJECT_NAME} unitTest"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_10}"
+          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+          disk_size: 100
+          disk_type: "pd-ssd"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+      - label: ":windows: Auditbeat Win-11 Unit Tests"
+        key: "auditbeat-extended-win-11-unit-tests"
+        command: "mage -d ${BEATS_PROJECT_NAME} unitTest"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_11}"
+          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+          disk_size: 100
+          disk_type: "pd-ssd"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+  - group: "Auditbeat Packaging"
+    key: "auditbeat-packaging"
+    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
+    depends_on:
+      - "auditbeat-mandatory-tests"
+
+    steps:
+      - label: ":ubuntu: Auditbeat/Packaging Linux X86"
+        key: "auditbeat-package-linux-x86"
+        env:
+          PLATFORMS: "${PACKAGING_PLATFORMS}"
+          SNAPSHOT: true
+        command: "cd $BEATS_PROJECT_NAME && mage package"
+        agents:
+          provider: gcp
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+
+      - label: ":linux: Auditbeat/Packaging Linux ARM"
+        key: "auditbeat-package-linux-arm"
+        env:
+          PLATFORMS: "${PACKAGING_ARM_PLATFORMS}"
+          PACKAGES: "docker"
+          SNAPSHOT: true
+        command: "cd $BEATS_PROJECT_NAME && mage package"
+        agents:
+          provider: "aws"
+          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
+          instanceType: "${AWS_ARM_INSTANCE_TYPE}"

.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml

@@ -4,62 +4,81 @@ env:
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
   GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
 
+notify:
+  - github_commit_status:
+      context: "Deploy/K8S"
+
 steps:
-  - label: "Checks"
-    command: ".buildkite/deploy/kubernetes/scripts/make.sh"
-    if: build.env("BUILDKITE_PULL_REQUEST") != "false" && build.env("GITHUB_PR_LABELS") == "kubernetes"
-    notify:
-      - github_commit_status:
-          context: "Deploy K8S/Checks"
-    agents:
-      provider: "gcp"
-      image: "${IMAGE_UBUNTU_X86_64}"
-      machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+  - group: "Depoy/K8S"
+    if: build.env("BUILDKITE_PULL_REQUEST") != "false" && build.env("GITHUB_PR_LABELS") =~ /.*kubernetes.*/
+
+    steps:
+    - label: "Checks"
+      command: ".buildkite/deploy/kubernetes/scripts/make.sh"
+      notify:
+        - github_commit_status:
+            context: "Deploy/k8s-checks"
+      agents:
+        provider: "gcp"
+        image: "${IMAGE_UBUNTU_X86_64}"
+        machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
 
-  - label: "K8S Test/K8S version: v1.29.0"
-    key: "k8s-test-129"
-    env:
-      K8S_VERSION: "v1.29.0"
-    commands:
-      - "MODULE=kubernetes make -C metricbeat integration-tests"
-      - "make -C deploy/kubernetes test"
-    agents:
-      provider: "gcp"
-      image: "${IMAGE_UBUNTU_X86_64}"
-      machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+    - label: "K8S Test/K8S version: v1.29.0"
+      key: "k8s-test-129"
+      env:
+        K8S_VERSION: "v1.29.0"
+      commands:
+        - "MODULE=kubernetes make -C metricbeat integration-tests"
+        - "make -C deploy/kubernetes test"
+      notify:
+        - github_commit_status:
+            context: "Deploy/k8s-test v1.29.0"
+      agents:
+        provider: "gcp"
+        image: "${IMAGE_UBUNTU_X86_64}"
+        machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
 
-  - label: "K8S Test/K8S version: v1.28.0"
-    key: "k8s-test-128"
-    env:
-      K8S_VERSION: "v1.28.0"
-    commands:
-      - "MODULE=kubernetes make -C metricbeat integration-tests"
-      - "make -C deploy/kubernetes test"
-    agents:
-      provider: "gcp"
-      image: "${IMAGE_UBUNTU_X86_64}"
-      machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+    - label: "K8S Test/K8S version: v1.28.0"
+      key: "k8s-test-128"
+      env:
+        K8S_VERSION: "v1.28.0"
+      commands:
+        - "MODULE=kubernetes make -C metricbeat integration-tests"
+        - "make -C deploy/kubernetes test"
+      notify:
+        - github_commit_status:
+            context: "Deploy/k8s-test v1.28.0"
+      agents:
+        provider: "gcp"
+        image: "${IMAGE_UBUNTU_X86_64}"
+        machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
 
-  - label: "K8S Test/K8S version: v1.27.3"
-    key: "k8s-test-1273"
-    env:
-      K8S_VERSION: "v1.27.3"
-    commands:
-      - "MODULE=kubernetes make -C metricbeat integration-tests"
-      - "make -C deploy/kubernetes test"
-    agents:
-      provider: "gcp"
-      image: "${IMAGE_UBUNTU_X86_64}"
-      machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+    - label: "K8S Test/K8S version: v1.27.3"
+      key: "k8s-test-1273"
+      env:
+        K8S_VERSION: "v1.27.3"
+      commands:
+        - "MODULE=kubernetes make -C metricbeat integration-tests"
+        - "make -C deploy/kubernetes test"
+      notify:
+        - github_commit_status:
+            context: "Deploy/k8s-test v1.27.3"
+      agents:
+        provider: "gcp"
+        image: "${IMAGE_UBUNTU_X86_64}"
+        machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
 
-  - label: "K8S Test/K8S version: v1.26.6"
-    key: "k8s-test-1266"
-    env:
-      K8S_VERSION: "v1.26.6"
-    commands:
-      - "MODULE=kubernetes make -C metricbeat integration-tests"
-      - "make -C deploy/kubernetes test"
-    agents:
-      provider: "gcp"
-      image: "${IMAGE_UBUNTU_X86_64}"
-      machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+    - label: "K8S Test/K8S version: v1.26.6"
+      key: "k8s-test-1266"
+      env:
+        K8S_VERSION: "v1.26.6"
+      commands:
+        - "MODULE=kubernetes make -C metricbeat integration-tests"
+        - "make -C deploy/kubernetes test"
+      notify:
+        - github_commit_status:
+            context: "Deploy/k8s-test v1.26.6"
+      agents:
+        provider: "gcp"
+        image: "${IMAGE_UBUNTU_X86_64}"
+        machineType: "${GCP_HI_PERF_MACHINE_TYPE}"