Merge branch 'main' into goal-convert-to-lens-gauge

.buildkite/pipeline-resource-definitions/fix-location-collection.ts

@@ -0,0 +1,68 @@
+#!/usr/bin/env ts-node-script
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+import fs from 'fs';
+import jsYaml from 'js-yaml';
+import path from 'path';
+import { execSync } from 'child_process';
+
+const EXCLUDE_LIST = ['locations.yml'];
+const REPO_FILES_BASE = 'https://github.com/elastic/kibana/blob/main';
+
+type BackstageLocationResource = object & {
+  spec: { targets: string[] };
+};
+
+async function main() {
+  const repoRoot = execSync('git rev-parse --show-toplevel').toString().trim();
+  const resourceDefinitionsFolder = path.resolve(
+    repoRoot,
+    '.buildkite',
+    'pipeline-resource-definitions'
+  );
+  const resourceDefinitionsBaseUrl = `${REPO_FILES_BASE}/.buildkite/pipeline-resource-definitions`;
+  const locationFile = path.resolve(resourceDefinitionsFolder, 'locations.yml');
+  const locationFileLines = fs.readFileSync(locationFile, 'utf8').split('\n');
+
+  const pipelines = readDirRecursively(resourceDefinitionsFolder)
+    .filter((file) => file.endsWith('.yml'))
+    .map((file) => file.replace(`${resourceDefinitionsFolder}/`, ''))
+    .filter((f) => !EXCLUDE_LIST.includes(f));
+
+  const preamble = locationFileLines.slice(0, 1);
+
+  const locationObj = jsYaml.load(
+    locationFileLines.slice(1).join('\n')
+  ) as BackstageLocationResource;
+  locationObj.spec.targets = pipelines.map(
+    (fileName) => `${resourceDefinitionsBaseUrl}/${fileName}`
+  );
+
+  const locationYaml = jsYaml.dump(locationObj, { lineWidth: 400 });
+
+  fs.writeFileSync(locationFile, `${preamble.join('\n')}\n${locationYaml}`);
+
+  console.log('Updated locations.yml');
+}
+
+function readDirRecursively(dir: string): string[] {
+  const files = fs.readdirSync(dir);
+  return files.reduce((acc, file) => {
+    const filePath = path.join(dir, file);
+    if (fs.statSync(filePath).isDirectory()) {
+      return [...acc, ...readDirRecursively(filePath)];
+    } else {
+      return [...acc, filePath];
+    }
+  }, [] as string[]);
+}
+
+main().catch((error) => {
+  console.error(error);
+  process.exit(1);
+});

.buildkite/pipeline-resource-definitions/kibana-coverage-daily.yml

@@ -0,0 +1,50 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: bk-kibana-code-coverage-main
+  description: 'Collects code coverage for unit and e2e tests, publishes results on Kibana stats cluster'
+  links:
+    - url: 'https://buildkite.com/elastic/kibana-code-coverage-main'
+      title: Pipeline link
+spec:
+  type: buildkite-pipeline
+  owner: 'group:appex-qa'
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana / code-coverage / main
+      description: 'Collects code coverage for unit and e2e tests, publishes results on Kibana stats cluster'
+    spec:
+      env:
+        SLACK_NOTIFICATIONS_CHANNEL: '#appex-qa-bots'
+        GITHUB_COMMIT_STATUS_CONTEXT: kibana-code-coverage-main
+        CODE_COVERAGE: '1'
+        FTR_CONFIGS_RETRY_COUNT: '0'
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+      allow_rebuilds: false
+      branch_configuration: main
+      default_branch: main
+      repository: elastic/kibana
+      pipeline_file: .buildkite/pipelines/code_coverage/daily.yml
+      skip_intermediate_builds: false
+      provider_settings:
+        prefix_pull_request_fork_branch_names: false
+        skip_pull_request_builds_for_existing_commits: true
+        trigger_mode: none
+      teams:
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        appex-qa:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-tech-leads:
+          access_level: MANAGE_BUILD_AND_READ
+        everyone:
+          access_level: BUILD_AND_READ
+      schedules:
+        Daily 6 am UTC:
+          cronline: 0 5 * * *
+          message: Daily 6 am UTC
+          branch: main

.buildkite/pipeline-resource-definitions/kibana-esql-grammar-sync.yml

@@ -0,0 +1,53 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: bk-kibana-esql-grammar-sync
+  description: Opens a PR if anything changes in the ES|QL grammar in Elasticsearch
+  links:
+    - url: 'https://buildkite.com/elastic/kibana-esql-grammar-sync'
+      title: Pipeline link
+spec:
+  type: buildkite-pipeline
+  owner: 'group:kibana-esql'
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana / ES|QL grammar sync
+      description: Opens a PR if anything changes in the ES|QL grammar in Elasticsearch
+    spec:
+      env:
+        SLACK_NOTIFICATIONS_CHANNEL: '#kibana-esql-project-team'
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+      allow_rebuilds: false
+      branch_configuration: main
+      default_branch: main
+      repository: elastic/kibana
+      pipeline_file: .buildkite/pipelines/esql_grammar_sync.yml
+      skip_intermediate_builds: false
+      provider_settings:
+        build_branches: false
+        build_pull_requests: false
+        publish_commit_status: false
+        trigger_mode: none
+        build_tags: false
+        prefix_pull_request_fork_branch_names: false
+        skip_pull_request_builds_for_existing_commits: true
+      teams:
+        kibana-esql:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        appex-qa:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-tech-leads:
+          access_level: MANAGE_BUILD_AND_READ
+        everyone:
+          access_level: BUILD_AND_READ
+      schedules:
+        Weekly build:
+          cronline: 0 0 * * 1 America/New_York
+          message: Weekly build
+          branch: main

.buildkite/pipeline-resource-definitions/kibana-purge-cloud-deployments.yml

@@ -0,0 +1,45 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: bk-kibana-purge-cloud-deployments
+  description: Purge stale cloud deployments
+  links:
+    - url: 'https://buildkite.com/elastic/kibana-purge-cloud-deployments'
+      title: Pipeline link
+spec:
+  type: buildkite-pipeline
+  owner: 'group:kibana-operations'
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana / purge-cloud-deployments
+      description: Purge stale cloud deployments
+    spec:
+      env:
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'false'
+      allow_rebuilds: false
+      branch_configuration: main
+      default_branch: main
+      repository: elastic/kibana
+      pipeline_file: .buildkite/pipelines/purge_cloud_deployments.yml
+      skip_intermediate_builds: false
+      provider_settings:
+        build_branches: false
+        build_pull_requests: false
+        publish_commit_status: false
+        trigger_mode: none
+        build_tags: false
+        prefix_pull_request_fork_branch_names: false
+        skip_pull_request_builds_for_existing_commits: true
+      teams:
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        appex-qa:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-tech-leads:
+          access_level: MANAGE_BUILD_AND_READ
+        everyone:
+          access_level: BUILD_AND_READ

.buildkite/pipeline-resource-definitions/locations.yml

@@ -6,13 +6,16 @@ metadata:
 spec:
   type: url
   targets:
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-coverage-daily.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-esql-grammar-sync.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-migration-staging.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-purge-cloud-deployments.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-release.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-ess/security-solution-ess.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-defend-workflows.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-detection-engine.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-entity-analytics.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-explore.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-gen-ai.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-investigations.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-rule-management.yml
-    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-detection-engine.yml
-    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-gen-ai.yml
-    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-entity-analytics.yml
-    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-defend-workflows.yml
-    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-ess/security-solution-ess.yml
-    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-release.yml

.buildkite/pipelines/code_coverage/daily.yml

@@ -3,14 +3,20 @@ steps:
     label: Pre-Build
     timeout_in_minutes: 10
     agents:
-      queue: kibana-default
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      machineType: n2-standard-2
 
   - wait
 
   - command: .buildkite/scripts/steps/test/pick_test_group_run_order.sh
     label: 'Pick Test Group Run Order'
     agents:
-      queue: kibana-default
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      machineType: n2-standard-2
     timeout_in_minutes: 10
     env:
       FTR_CONFIGS_DEPS: ''
@@ -21,7 +27,11 @@ steps:
   - command: .buildkite/scripts/steps/code_coverage/ingest.sh
     label: 'Merge and Ingest'
     agents:
-      queue: n2-4-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
     depends_on:
       - jest
       - jest-integration

.buildkite/pipelines/esql_grammar_sync.yml

@@ -3,4 +3,8 @@ steps:
     label: ES|QL Grammar Sync
     timeout_in_minutes: 10
     agents:
-      queue: n2-2-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      machineType: n2-standard-2
+      preemptible: true

.buildkite/pipelines/pull_request/apm_cypress.yml

@@ -7,7 +7,7 @@ steps:
       - build
       - quick_checks
     timeout_in_minutes: 120
-    parallelism: 4
+    parallelism: 1 # TODO: Set parallelism when apm_cypress handles it
     retry:
       automatic:
         - exit_status: '-1'

.buildkite/pipelines/purge_cloud_deployments.yml

@@ -3,4 +3,7 @@ steps:
     label: Purge old cloud deployments
     timeout_in_minutes: 10
     agents:
-      queue: kibana-default
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      machineType: n2-standard-2