[7.17] Merge pull request #318 from elastic/buildkite-migration

.buildkite/branches.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+# This determines which branches will have pipelines triggered periodically, for dra workflows.
+BRANCHES=(main 8.10 8.9 7.17)

.buildkite/dra-build.ps1

@@ -0,0 +1,18 @@
+New-Item C:/.cert -Type Directory -Force
+[IO.File]::WriteAllText("C:/.cert/msi_certificate.p12", $Env:MSI_CERTIFICATE_SECRET)
+[IO.File]::WriteAllText("C:/.cert/msi_password.txt", $Env:MSI_PASSWORD)
+
+Select-String -Pattern '^set\(DRV_VERSION (.*)\)$' -Path ".\CMakeLists.txt" | Select-Object -ExpandProperty Matches -First 1 -OutVariable Match
+$VersionString=$Match.groups[1].Value
+
+if ($Env:DRA_WORKFLOW -eq "staging") {
+	cmd.exe /c 'build.bat setup proper'
+	cmd.exe /c 'build.bat setup 64 type:Release package sign:C:/.cert/msi_certificate.p12+C:/.cert/msi_password.txt'
+	cmd.exe /c 'build.bat setup 32 type:Release package sign:C:/.cert/msi_certificate.p12+C:/.cert/msi_password.txt'
+} else {
+	cmd.exe /c 'build.bat setup proper'
+	cmd.exe /c 'build.bat setup 64 type:Release package:-SNAPSHOT sign:C:/.cert/msi_certificate.p12+C:/.cert/msi_password.txt'
+	cmd.exe /c 'build.bat setup 32 type:Release package:-SNAPSHOT sign:C:/.cert/msi_certificate.p12+C:/.cert/msi_password.txt'
+}
+
+buildkite-agent artifact upload 'installer/build/out/*.msi'

.buildkite/dra-release.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+set -euo pipefail
+
+DRA_WORKFLOW=${DRA_WORKFLOW:-snapshot}
+
+if [[ "$BUILDKITE_BRANCH" == "main" && "$DRA_WORKFLOW" == "staging" ]]; then
+	exit 0
+fi
+
+buildkite-agent artifact download '*.msi' .
+
+RM_BRANCH="$BUILDKITE_BRANCH"
+if [[ "$BUILDKITE_BRANCH" == "main" ]]; then
+	RM_BRANCH=master
+fi
+
+DRV_VERSION=$(grep 'set(DRV_VERSION' CMakeLists.txt | grep -o '[0-9]\+\.[0-9]\+\.[0-9]\+')
+GIT_COMMIT=$(git rev-parse HEAD)
+
+# Allow other users access to read the artifacts so they are readable in the container
+chmod a+r installer/build/out/*
+
+# Allow other users write access to create checksum files
+chmod a+w installer/build/out
+
+docker run --rm \
+	--name release-manager \
+	-e VAULT_ADDR="$DRA_VAULT_ADDR" \
+	-e VAULT_ROLE_ID="$DRA_VAULT_ROLE_ID_SECRET" \
+	-e VAULT_SECRET_ID="$DRA_VAULT_SECRET_ID_SECRET" \
+	--mount type=bind,readonly=false,src="$PWD",target=/artifacts \
+	docker.elastic.co/infra/release-manager:latest \
+	cli collect \
+	--project elasticsearch-sql-odbc \
+	--branch "$RM_BRANCH" \
+	--commit "$GIT_COMMIT" \
+	--workflow "$DRA_WORKFLOW" \
+	--version "$DRV_VERSION" \
+	--artifact-set main

.buildkite/dra-workflow.trigger.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -euo pipefail
+
+echo "steps:"
+
+source .buildkite/branches.sh
+
+for BRANCH in "${BRANCHES[@]}"; do
+  cat <<EOF
+  - trigger: elasticsearch-sql-odbc-dra-workflow
+    label: Trigger DRA snapshot workflow for $BRANCH
+    async: true
+    build:
+      branch: $BRANCH
+      env:
+        DRA_WORKFLOW: snapshot
+EOF
+
+	# Don't trigger staging workflow for main branch
+	if [[ "$BRANCH" != "main" ]]; then
+		cat <<EOF
+  - trigger: elasticsearch-sql-odbc-dra-workflow
+    label: Trigger DRA staging workflow for $BRANCH
+    async: true
+    build:
+      branch: $BRANCH
+      env:
+        DRA_WORKFLOW: staging
+EOF
+	fi
+done

.buildkite/hooks/pre-command

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+if [[ "$USE_DRA_CREDENTIALS" == "true" ]]; then
+  DRA_VAULT_ROLE_ID_SECRET=$(vault read -field=role-id secret/ci/elastic-elasticsearch-sql-odbc/legacy-vault-credentials)
+  export DRA_VAULT_ROLE_ID_SECRET
+
+  DRA_VAULT_SECRET_ID_SECRET=$(vault read -field=secret-id secret/ci/elastic-elasticsearch-sql-odbc/legacy-vault-credentials)
+  export DRA_VAULT_SECRET_ID_SECRET
+
+  DRA_VAULT_ADDR=https://secrets.elastic.co:8200
+  export DRA_VAULT_ADDR
+fi

.buildkite/hooks/pre-command.ps1

@@ -0,0 +1,2 @@
+$env:MSI_CERTIFICATE_SECRET = vault read -field=cert secret/ci/elastic-elasticsearch-sql-odbc/msi
+$env:MSI_PASSWORD = vault read -field=password secret/ci/elastic-elasticsearch-sql-odbc/msi

.buildkite/pipeline.dra.yml

@@ -0,0 +1,18 @@
+steps:
+  - label: Build artifacts
+    command: powershell .buildkite\dra-build.ps1
+    agents:
+      provider: gcp
+      image: family/elasticsearch-windows-2022
+      machineType: n2-standard-8
+      diskSizeGb: 200
+  - wait
+  - label: Publish artifacts
+    command: .buildkite/dra-release.sh
+    env:
+      USE_DRA_CREDENTIALS: "true"
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: n2-standard-4
+      diskSizeGb: 100

.buildkite/pipeline.yml

@@ -0,0 +1,8 @@
+steps:
+  - label: Run tests
+    command: cmd /c build.bat ctests
+    agents:
+      provider: gcp
+      image: family/elasticsearch-windows-2022
+      machineType: n2-standard-8
+      diskSizeGb: 200

.editorconfig

@@ -16,7 +16,7 @@ indent_size = 4
 indent_style = space
 indent_size = 4
 
-[*.{md,markdown,json,js,csproj,fsproj,targets,targets,props}]
+[*.{md,markdown,json,js,csproj,fsproj,targets,targets,props,yml,yaml}]
 indent_style = space
 indent_size = 2
 

catalog-info.yaml

@@ -0,0 +1,112 @@
+---
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: buildkite-pipeline-elasticsearch-sql-odbc
+  description: On-merge pipeline for elasticsearch-sql-odbc tests
+  links:
+    - title: Pipeline
+      url: https://buildkite.com/elastic/elasticsearch-sql-odbc
+spec:
+  type: buildkite-pipeline
+  owner: group:elasticsearch-team
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: elasticsearch-sql-odbc
+    spec:
+      repository: elastic/elasticsearch-sql-odbc
+      pipeline_file: .buildkite/pipeline.yml
+      default_branch: main
+      branch_configuration: "main 8.* 7.17"
+      provider_settings:
+        trigger_mode: code
+        build_pull_requests: false
+        build_pull_request_forks: false
+        publish_commit_status: false
+        build_branches: true
+      teams:
+        elasticsearch-team:
+          access_level: MANAGE_BUILD_AND_READ
+        everyone:
+          access_level: BUILD_AND_READ
+---
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: buildkite-pipeline-elasticsearch-sql-odbc-dra-workflow
+  description: Publishes DRA snapshot or staging candidate
+  links:
+    - title: Pipeline
+      url: https://buildkite.com/elastic/elasticsearch-sql-odbc-dra-workflow
+spec:
+  type: buildkite-pipeline
+  system: buildkite
+  owner: group:elasticsearch-team
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      description: "Publishes DRA snapshot or staging candidate"
+      name: elasticsearch-sql-odbc / dra-workflow
+    spec:
+      repository: elastic/elasticsearch-sql-odbc
+      pipeline_file: .buildkite/pipeline.dra.yml
+      env:
+        ENABLE_DRA_WORKFLOW: "true"
+      default_branch: main
+      branch_configuration: "main 8.* 7.17"
+      teams:
+        elasticsearch-team: {}
+        everyone:
+          access_level: BUILD_AND_READ
+      provider_settings:
+        trigger_mode: none
+        build_pull_requests: false
+        build_pull_request_forks: false
+        publish_commit_status: false
+        build_branches: false
+---
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: buildkite-pipeline-elasticsearch-sql-odbc-dra-workflow-trigger
+  description: Triggers piplines for DRA workflows for all release branches
+  links:
+    - title: Pipeline
+      url: https://buildkite.com/elastic/elasticsearch-sql-odbc-dra-workflow-trigger
+spec:
+  type: buildkite-pipeline
+  system: buildkite
+  owner: group:elasticsearch-team
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      description: Triggers piplines for DRA workflows for all release branches
+      name: elasticsearch-sql-odbc / dra-workflow-trigger
+    spec:
+      repository: elastic/elasticsearch-sql-odbc
+      pipeline_file: .buildkite/dra-workflow.trigger.sh
+      default_branch: main
+      branch_configuration: main
+      teams:
+        elasticsearch-team: {}
+        everyone:
+          access_level: BUILD_AND_READ
+      provider_settings:
+        trigger_mode: none
+        build_pull_requests: false
+        build_pull_request_forks: false
+        publish_commit_status: false
+        build_branches: false
+      schedules:
+        Periodically on main:
+          branch: main
+          cronline: "0 0,12 * * * America/New_York"
+          message: "Triggers DRA workflows 2x per day"