Merge branch 'main' into test_discoverability

elastic · May 13, 2024 · 6f39ead · 6f39ead
2 parents 5c90a94 + 931b587
commit 6f39ead
Show file tree

Hide file tree

Showing 128 changed files with 3,414 additions and 1,047 deletions.
diff --git a/.buildkite/pipeline-resource-definitions/kibana-artifacts-container-image.yml b/.buildkite/pipeline-resource-definitions/kibana-artifacts-container-image.yml
@@ -0,0 +1,44 @@
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: bk-kibana-artifacts-container-image
+  description: Kibana container image artifact builds
+  links:
+    - url: https://buildkite.com/elastic/kibana-artifacts-container-image
+      title: Pipeline link
+spec:
+  type: buildkite-pipeline
+  owner: group:kibana-operations
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana / artifacts container image
+      description: Kibana container image artifact builds
+    spec:
+      env:
+        SLACK_NOTIFICATIONS_CHANNEL: '#kibana-serverless-test-alerts'
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+      allow_rebuilds: true
+      branch_configuration: main
+      repository: elastic/kibana
+      pipeline_file: .buildkite/pipelines/artifacts_container_image.yml
+      skip_intermediate_builds: false
+      provider_settings:
+        build_branches: false
+        build_pull_requests: false
+        publish_commit_status: false
+        trigger_mode: none
+        build_tags: false
+        prefix_pull_request_fork_branch_names: false
+        skip_pull_request_builds_for_existing_commits: false
+      teams:
+        everyone:
+          access_level: BUILD_AND_READ
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        appex-qa:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-tech-leads:
+          access_level: MANAGE_BUILD_AND_READ
diff --git a/.buildkite/pipeline-resource-definitions/kibana-artifacts-snapshot.yml b/.buildkite/pipeline-resource-definitions/kibana-artifacts-snapshot.yml
@@ -0,0 +1,58 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: bk-kibana-artifacts-snapshot
+  description: Kibana snapshot artifact builds
+  links:
+    - url: https://buildkite.com/elastic/kibana-artifacts-snapshot
+      title: Pipeline link
+spec:
+  type: buildkite-pipeline
+  owner: group:kibana-operations
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana / artifacts snapshot
+      description: Kibana snapshot artifact builds
+    spec:
+      env:
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+      allow_rebuilds: true
+      branch_configuration: main 8.14 7.17
+      default_branch: main
+      repository: elastic/kibana
+      pipeline_file: .buildkite/pipelines/artifacts.yml
+      skip_intermediate_builds: false
+      provider_settings:
+        build_branches: false
+        build_pull_requests: false
+        publish_commit_status: false
+        trigger_mode: none
+        build_tags: false
+        prefix_pull_request_fork_branch_names: false
+        skip_pull_request_builds_for_existing_commits: false
+      teams:
+        everyone:
+          access_level: BUILD_AND_READ
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        appex-qa:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-tech-leads:
+          access_level: MANAGE_BUILD_AND_READ
+      schedules:
+        Daily build (main):
+          cronline: 0 7 * * * America/New_York
+          message: Daily build
+          branch: main
+        Daily build (8.14):
+          cronline: 0 7 * * * America/New_York
+          message: Daily build
+          branch: '8.14'
+        Daily build (7.17):
+          cronline: 0 7 * * * America/New_York
+          message: Daily build
+          branch: '7.17'
diff --git a/.buildkite/pipeline-resource-definitions/kibana-artifacts-staging.yml b/.buildkite/pipeline-resource-definitions/kibana-artifacts-staging.yml
@@ -0,0 +1,54 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: bk-kibana-artifacts-staging
+  description: Kibana staging artifact builds
+  links:
+    - url: https://buildkite.com/elastic/kibana-artifacts-staging
+      title: Pipeline link
+spec:
+  type: buildkite-pipeline
+  owner: group:kibana-operations
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana / artifacts staging
+      description: Kibana staging artifact builds
+    spec:
+      env:
+        RELEASE_BUILD: 'true'
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+      allow_rebuilds: true
+      branch_configuration: 7.17 8.14
+      repository: elastic/kibana
+      pipeline_file: .buildkite/pipelines/artifacts.yml
+      skip_intermediate_builds: false
+      provider_settings:
+        build_branches: false
+        build_pull_requests: false
+        publish_commit_status: false
+        trigger_mode: none
+        build_tags: false
+        prefix_pull_request_fork_branch_names: false
+        skip_pull_request_builds_for_existing_commits: false
+      teams:
+        everyone:
+          access_level: BUILD_AND_READ
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        appex-qa:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-tech-leads:
+          access_level: MANAGE_BUILD_AND_READ
+      schedules:
+        Daily build (8.14):
+          cronline: 0 7 * * * America/New_York
+          message: Daily build
+          branch: '8.14'
+        Daily build (7.17):
+          cronline: 0 7 * * * America/New_York
+          message: Daily build
+          branch: '7.17'
diff --git a/.buildkite/pipeline-resource-definitions/kibana-artifacts-trigger.yml b/.buildkite/pipeline-resource-definitions/kibana-artifacts-trigger.yml
@@ -0,0 +1,51 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: bk-kibana-artifacts-trigger
+  description: Kibana artifact trigger
+  links:
+    - url: 'https://buildkite.com/elastic/kibana-artifacts-trigger'
+      title: Pipeline link
+spec:
+  type: buildkite-pipeline
+  owner: 'group:kibana-operations'
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana / artifacts trigger
+      description: Kibana artifact trigger
+    spec:
+      env:
+        RELEASE_BUILD: 'true'
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+      allow_rebuilds: true
+      branch_configuration: '8.14'
+      default_branch: main
+      repository: elastic/kibana
+      pipeline_file: .buildkite/pipelines/artifacts_trigger.yml
+      skip_intermediate_builds: false
+      provider_settings:
+        build_branches: false
+        build_pull_requests: false
+        publish_commit_status: false
+        trigger_mode: none
+        build_tags: false
+        prefix_pull_request_fork_branch_names: false
+        skip_pull_request_builds_for_existing_commits: false
+      teams:
+        everyone:
+          access_level: BUILD_AND_READ
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        appex-qa:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-tech-leads:
+          access_level: MANAGE_BUILD_AND_READ
+      schedules:
+        Daily build (8.14):
+          cronline: 0 */2 * * * America/New_York
+          message: Daily build
+          branch: '8.14'
diff --git a/.buildkite/pipeline-resource-definitions/locations.yml b/.buildkite/pipeline-resource-definitions/locations.yml
@@ -9,6 +9,10 @@ spec:
   targets:
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-api-docs.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-apis-capacity-testing-daily.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-artifacts-container-image.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-artifacts-snapshot.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-artifacts-staging.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-artifacts-trigger.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-coverage-daily.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-serverless-snapshots.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-snapshots.yml

diff --git a/.buildkite/pipelines/artifacts.yml b/.buildkite/pipelines/artifacts.yml
@@ -2,7 +2,10 @@ steps:
   - command: .buildkite/scripts/steps/artifacts/build.sh
     label: Build Kibana Artifacts
     agents:
-      queue: c2-16
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      machineType: c2-standard-16
     timeout_in_minutes: 60
     retry:
       automatic:
@@ -14,7 +17,13 @@ steps:
   - command: TEST_PACKAGE=deb .buildkite/scripts/steps/package_testing/test.sh
     label: Artifact Testing
     agents:
-      queue: n2-4-virt
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      enableNestedVirtualization: true
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: n2-standard-4
     timeout_in_minutes: 30
     retry:
       automatic:
@@ -24,7 +33,13 @@ steps:
   - command: TEST_PACKAGE=rpm .buildkite/scripts/steps/package_testing/test.sh
     label: Artifact Testing
     agents:
-      queue: n2-4-virt
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      enableNestedVirtualization: true
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: n2-standard-4
     timeout_in_minutes: 30
     retry:
       automatic:
@@ -34,7 +49,13 @@ steps:
   - command: TEST_PACKAGE=docker .buildkite/scripts/steps/package_testing/test.sh
     label: Artifact Testing
     agents:
-      queue: n2-4-virt
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      enableNestedVirtualization: true
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: n2-standard-4
     timeout_in_minutes: 30
     retry:
       automatic:
@@ -44,7 +65,12 @@ steps:
   - command: KIBANA_DOCKER_CONTEXT=default .buildkite/scripts/steps/artifacts/docker_context.sh
     label: 'Docker Context Verification'
     agents:
-      queue: n2-2
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: n2-standard-2
     timeout_in_minutes: 30
     retry:
       automatic:
@@ -54,7 +80,12 @@ steps:
   - command: KIBANA_DOCKER_CONTEXT=ubi .buildkite/scripts/steps/artifacts/docker_context.sh
     label: 'Docker Context Verification'
     agents:
-      queue: n2-2
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: n2-standard-2
     timeout_in_minutes: 30
     retry:
       automatic:
@@ -64,7 +95,12 @@ steps:
   - command: KIBANA_DOCKER_CONTEXT=ironbank .buildkite/scripts/steps/artifacts/docker_context.sh
     label: 'Docker Context Verification'
     agents:
-      queue: n2-2
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: n2-standard-2
     timeout_in_minutes: 30
     retry:
       automatic:
@@ -77,7 +113,12 @@ steps:
       - exit_status: 255
       - exit_status: -1
     agents:
-      queue: n2-2
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: n2-standard-2
     timeout_in_minutes: 60
     if: "build.env('RELEASE_BUILD') == null || build.env('RELEASE_BUILD') == '' || build.env('RELEASE_BUILD') == 'false'"
     retry:
@@ -99,5 +140,10 @@ steps:
   - command: .buildkite/scripts/steps/artifacts/publish.sh
     label: 'Publish Kibana Artifacts'
     agents:
-      queue: n2-2
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: n2-standard-2
     timeout_in_minutes: 30
diff --git a/.buildkite/pipelines/artifacts_container_image.yml b/.buildkite/pipelines/artifacts_container_image.yml
@@ -2,5 +2,8 @@ steps:
   - command: .buildkite/scripts/steps/artifacts/docker_image.sh
     label: Build serverless container images
     agents:
-      queue: c2-16
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      machineType: n2-standard-16
     timeout_in_minutes: 60
diff --git a/.buildkite/pipelines/artifacts_trigger.yml b/.buildkite/pipelines/artifacts_trigger.yml
@@ -2,5 +2,8 @@ steps:
   - command: .buildkite/scripts/steps/artifacts/trigger.sh
     label: Trigger artifacts build
     agents:
-      queue: kibana-default
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      machineType: n2-standard-2
     timeout_in_minutes: 10
diff --git a/.buildkite/scripts/build_kibana.sh b/.buildkite/scripts/build_kibana.sh
@@ -20,7 +20,6 @@ echo "> node scripts/build" "${BUILD_ARGS[@]}"
 node scripts/build "${BUILD_ARGS[@]}"
 
 if is_pr_with_label "ci:build-cloud-image"; then
-  echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co
   node scripts/build \
   --skip-initialize \
   --skip-generic-folders \
@@ -35,7 +34,6 @@ if is_pr_with_label "ci:build-cloud-image"; then
   --skip-docker-ubuntu \
   --skip-docker-serverless \
   --skip-docker-contexts
-  docker logout docker.elastic.co
 
   CLOUD_IMAGE=$(docker images --format "{{.Repository}}:{{.Tag}}" docker.elastic.co/kibana-ci/kibana-cloud)
   cat << EOF | buildkite-agent annotate --style "info" --context kibana-cloud-image

diff --git a/.buildkite/scripts/common/setup_job_env.sh b/.buildkite/scripts/common/setup_job_env.sh
@@ -20,10 +20,10 @@ fi
   export KIBANA_CI_GITHUB_TOKEN
 
   KIBANA_DOCKER_USERNAME="$(vault_get container-registry username)"
-  export KIBANA_DOCKER_USERNAME
-
   KIBANA_DOCKER_PASSWORD="$(vault_get container-registry password)"
-  export KIBANA_DOCKER_PASSWORD
+  if command -v docker &> /dev/null; then
+    echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co
+  fi
 }
 
 # Set up a custom ES Snapshot Manifest if one has been specified for this build