[buildkite] Add third-party tests to periodic pipeline (#99376) (#100094

)

(cherry picked from commit f69dce0)

.buildkite/hooks/pre-command

@@ -48,7 +48,7 @@ BUILDKITE_API_TOKEN=$(vault read -field=token secret/ci/elastic-elasticsearch/bu
 export BUILDKITE_API_TOKEN
 
 if [[ "${USE_LUCENE_SNAPSHOT_CREDS:-}" == "true" ]]; then
-  data=$(.buildkite/scripts/lucene-snapshot/get-credentials.sh)
+  data=$(.buildkite/scripts/get-legacy-secret.sh aws-elastic/creds/lucene-snapshots)
 
   AWS_ACCESS_KEY_ID=$(echo "$data" | jq -r .data.access_key)
   export AWS_ACCESS_KEY_ID
@@ -70,6 +70,8 @@ if [[ "${USE_DRA_CREDENTIALS:-}" == "true" ]]; then
   export DRA_VAULT_ADDR
 fi
 
+source .buildkite/scripts/third-party-test-credentials.sh
+
 if [[ "${USE_SNYK_CREDENTIALS:-}" == "true" ]]; then
   SNYK_TOKEN=$(vault read -field=token secret/ci/elastic-elasticsearch/migrated/snyk)
   export SNYK_TOKEN

.buildkite/pipelines/periodic.template.yml

@@ -103,6 +103,73 @@ steps:
       image: family/elasticsearch-ubuntu-2004
       diskSizeGb: 350
       machineType: custom-32-98304
+  - group: third-party tests
+    steps:
+      - label: third-party / azure-sas
+        command: |
+          export azure_storage_container=elasticsearch-ci-thirdparty-sas
+          export azure_storage_base_path=$BUILDKITE_BRANCH
+
+          .ci/scripts/run-gradle.sh azureThirdPartyTest
+        env:
+          USE_3RD_PARTY_AZURE_SAS_CREDENTIALS: "true"
+        timeout_in_minutes: 30
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n2-standard-8
+          buildDirectory: /dev/shm/bk
+      - label: third-party / azure
+        command: |
+          export azure_storage_container=elasticsearch-ci-thirdparty
+          export azure_storage_base_path=$BUILDKITE_BRANCH
+
+          .ci/scripts/run-gradle.sh azureThirdPartyTest
+        env:
+          USE_3RD_PARTY_AZURE_CREDENTIALS: "true"
+        timeout_in_minutes: 30
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n2-standard-8
+          buildDirectory: /dev/shm/bk
+      - label: third-party / gcs
+        command: |
+          export google_storage_bucket=elasticsearch-ci-thirdparty
+          export google_storage_base_path=$BUILDKITE_BRANCH
+
+          .ci/scripts/run-gradle.sh gcsThirdPartyTest
+        env:
+          USE_3RD_PARTY_GCS_CREDENTIALS: "true"
+        timeout_in_minutes: 30
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n2-standard-8
+          buildDirectory: /dev/shm/bk
+      - label: third-party / geoip
+        command: |
+          .ci/scripts/run-gradle.sh :modules:ingest-geoip:internalClusterTest -Dtests.jvm.argline="-Dgeoip_use_service=true"
+        timeout_in_minutes: 30
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n2-standard-8
+          buildDirectory: /dev/shm/bk
+      - label: third-party / s3
+        command: |
+          export amazon_s3_bucket=elasticsearch-ci.us-west-2
+          export amazon_s3_base_path=$BUILDKITE_BRANCH
+
+          .ci/scripts/run-gradle.sh s3ThirdPartyTest
+        env:
+          USE_3RD_PARTY_S3_CREDENTIALS: "true"
+        timeout_in_minutes: 30
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n2-standard-8
+          buildDirectory: /dev/shm/bk
   - label: Upload Snyk Dependency Graph
     command: .ci/scripts/run-gradle.sh uploadSnykDependencyGraph -PsnykTargetReference=$BUILDKITE_BRANCH
     env:

.buildkite/pipelines/periodic.yml

@@ -1114,6 +1114,72 @@ steps:
       image: family/elasticsearch-ubuntu-2004
       diskSizeGb: 350
       machineType: custom-32-98304
+  - group: third-party tests
+    steps:
+      - label: third-party / azure-sas
+        command: |
+          export azure_storage_container=elasticsearch-ci-thirdparty-sas
+          export azure_storage_base_path=$BUILDKITE_BRANCH
+
+          .ci/scripts/run-gradle.sh azureThirdPartyTest
+        env:
+          USE_3RD_PARTY_AZURE_SAS_CREDENTIALS: "true"
+        timeout_in_minutes: 30
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n2-standard-8
+          buildDirectory: /dev/shm/bk
+      - label: third-party / azure
+        command: |
+          export azure_storage_container=elasticsearch-ci-thirdparty
+          export azure_storage_base_path=$BUILDKITE_BRANCH
+
+          .ci/scripts/run-gradle.sh azureThirdPartyTest
+        env:
+          USE_3RD_PARTY_AZURE_CREDENTIALS: "true"
+        timeout_in_minutes: 30
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n2-standard-8
+          buildDirectory: /dev/shm/bk
+      - label: third-party / gcs
+        command: |
+          export google_storage_bucket=elasticsearch-ci-thirdparty
+          export google_storage_base_path=$BUILDKITE_BRANCH
+
+          .ci/scripts/run-gradle.sh gcsThirdPartyTest
+        env:
+          USE_3RD_PARTY_GCS_CREDENTIALS: "true"
+        timeout_in_minutes: 30
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n2-standard-8
+          buildDirectory: /dev/shm/bk
+      - label: third-party / geoip
+        command: |
+          .ci/scripts/run-gradle.sh :modules:ingest-geoip:internalClusterTest -Dtests.jvm.argline="-Dgeoip_use_service=true"
+        timeout_in_minutes: 30
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n2-standard-8
+      - label: third-party / s3
+        command: |
+          export amazon_s3_bucket=elasticsearch-ci.us-west-2
+          export amazon_s3_base_path=$BUILDKITE_BRANCH
+
+          .ci/scripts/run-gradle.sh s3ThirdPartyTest
+        env:
+          USE_3RD_PARTY_S3_CREDENTIALS: "true"
+        timeout_in_minutes: 30
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n2-standard-8
+          buildDirectory: /dev/shm/bk
   - label: Upload Snyk Dependency Graph
     command: .ci/scripts/run-gradle.sh uploadSnykDependencyGraph -PsnykTargetReference=$BUILDKITE_BRANCH
     env:

.buildkite/scripts/get-legacy-secret.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# WARNING: this script will echo the credentials to the console. It is meant to be called from another script and captured in a variable.
+# It should really only be used inside .buildkite/hooks/pre-command
+
+source .buildkite/scripts/setup-legacy-vault.sh
+
+vault read -format=json "$1"

.buildkite/scripts/lucene-snapshot/get-credentials.sh → .buildkite/scripts/setup-legacy-vault.sh

@@ -2,15 +2,10 @@
 
 set -euo pipefail
 
-# WARNING: this script will echo the credentials to the console. It is meant to be called from another script and captured in a variable.
-# It should really only be used inside .buildkite/hooks/pre-command
-
 VAULT_ROLE_ID=$(vault read -field=role-id secret/ci/elastic-elasticsearch/legacy-vault-credentials)
 VAULT_SECRET_ID=$(vault read -field=secret-id secret/ci/elastic-elasticsearch/legacy-vault-credentials)
 VAULT_ADDR=https://secrets.elastic.co:8200
 
 unset VAULT_TOKEN
 VAULT_TOKEN=$(vault write -field=token auth/approle/login role_id=$VAULT_ROLE_ID secret_id=$VAULT_SECRET_ID)
 export VAULT_TOKEN
-
-vault read -format=json aws-elastic/creds/lucene-snapshots

.buildkite/scripts/third-party-test-credentials.gcs.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# Usage: .buildkite/scripts/third-party-test-credentials.gcs.sh <path/to/write/credentials.json>
+
+source .buildkite/scripts/setup-legacy-vault.sh
+
+vault read -field=private_key_data gcp-elastic-ci-prod/key/elasticsearch-ci-thirdparty-gcs | base64 --decode > "$1"

.buildkite/scripts/third-party-test-credentials.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# You'll notice that most of the variables are exported twice with different names here
+# The first/uppercase export is to ensure that Buildkite masks the values in the logs should they accidentally be output
+# The second/lowercase export is what the tests expect/require
+
+if [[ "${USE_3RD_PARTY_AZURE_CREDENTIALS:-}" == "true" ]]; then
+  json=$(vault read -format=json secret/ci/elastic-elasticsearch/migrated/azure_thirdparty_test_creds)
+
+  AZURE_STORAGE_ACCOUNT_SECRET=$(echo "$json" | jq -r .data.account_id)
+  export AZURE_STORAGE_ACCOUNT_SECRET
+  export azure_storage_account="$AZURE_STORAGE_ACCOUNT_SECRET"
+
+  AZURE_STORAGE_KEY=$(echo "$json" | jq -r .data.account_key)
+  export AZURE_STORAGE_KEY
+  export azure_storage_key="$AZURE_STORAGE_KEY"
+fi
+
+if [[ "${USE_3RD_PARTY_AZURE_SAS_CREDENTIALS:-}" == "true" ]]; then
+  json=$(vault read -format=json secret/ci/elastic-elasticsearch/migrated/azure_thirdparty_sas_test_creds)
+
+  AZURE_STORAGE_ACCOUNT_SECRET=$(echo "$json" | jq -r .data.account_id)
+  export AZURE_STORAGE_ACCOUNT_SECRET
+  export azure_storage_account="$AZURE_STORAGE_ACCOUNT_SECRET"
+
+  AZURE_STORAGE_SAS_TOKEN=$(echo "$json" | jq -r .data.account_sas_token)
+  export AZURE_STORAGE_SAS_TOKEN
+  export azure_storage_sas_token="$AZURE_STORAGE_SAS_TOKEN"
+fi
+
+if [[ "${USE_3RD_PARTY_S3_CREDENTIALS:-}" == "true" ]]; then
+  json=$(.buildkite/scripts/get-legacy-secret.sh aws-test/creds/elasticsearch-ci-s3)
+  AMAZON_S3_ACCESS_KEY=$(echo "$json" | jq -r .data.access_key)
+  export AMAZON_S3_ACCESS_KEY
+  export amazon_s3_access_key="$AMAZON_S3_ACCESS_KEY"
+
+  AMAZON_S3_SECRET_KEY=$(echo "$json" | jq -r .data.secret_key)
+  export AMAZON_S3_SECRET_KEY
+  export amazon_s3_secret_key="$AMAZON_S3_SECRET_KEY"
+fi
+
+if [[ "${USE_3RD_PARTY_GCS_CREDENTIALS:-}" == "true" ]]; then
+  export google_storage_service_account=$(mktemp)
+  .buildkite/scripts/third-party-test-credentials.gcs.sh "$google_storage_service_account"
+fi
+
+
+
+unset json

test/fixtures/geoip-fixture/src/main/java/fixture/geoip/GeoIpHttpFixture.java

@@ -25,6 +25,7 @@
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
 
 public class GeoIpHttpFixture extends ExternalResource {
 
@@ -106,15 +107,25 @@ protected void after() {
     }
 
     private void copyFiles() throws Exception {
-        Files.copy(GeoIpHttpFixture.class.getResourceAsStream("/geoip-fixture/GeoLite2-ASN.tgz"), source.resolve("GeoLite2-ASN.tgz"));
-        Files.copy(GeoIpHttpFixture.class.getResourceAsStream("/geoip-fixture/GeoLite2-City.mmdb"), source.resolve("GeoLite2-City.mmdb"));
+        Files.copy(
+            GeoIpHttpFixture.class.getResourceAsStream("/geoip-fixture/GeoLite2-ASN.tgz"),
+            source.resolve("GeoLite2-ASN.tgz"),
+            StandardCopyOption.REPLACE_EXISTING
+        );
+        Files.copy(
+            GeoIpHttpFixture.class.getResourceAsStream("/geoip-fixture/GeoLite2-City.mmdb"),
+            source.resolve("GeoLite2-City.mmdb"),
+            StandardCopyOption.REPLACE_EXISTING
+        );
         Files.copy(
             GeoIpHttpFixture.class.getResourceAsStream("/geoip-fixture/GeoLite2-Country.mmdb"),
-            source.resolve("GeoLite2-Country.mmdb")
+            source.resolve("GeoLite2-Country.mmdb"),
+            StandardCopyOption.REPLACE_EXISTING
         );
         Files.copy(
             GeoIpHttpFixture.class.getResourceAsStream("/geoip-fixture/MyCustomGeoLite2-City.mmdb"),
-            source.resolve("MyCustomGeoLite2-City.mmdb")
+            source.resolve("MyCustomGeoLite2-City.mmdb"),
+            StandardCopyOption.REPLACE_EXISTING
         );
 
         new GeoIpCli().main(