diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index f244b0b51c3e9..40fb970a76196 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -48,7 +48,7 @@ BUILDKITE_API_TOKEN=$(vault read -field=token secret/ci/elastic-elasticsearch/bu export BUILDKITE_API_TOKEN if [[ "${USE_LUCENE_SNAPSHOT_CREDS:-}" == "true" ]]; then - data=$(.buildkite/scripts/lucene-snapshot/get-credentials.sh) + data=$(.buildkite/scripts/get-legacy-secret.sh aws-elastic/creds/lucene-snapshots) AWS_ACCESS_KEY_ID=$(echo "$data" | jq -r .data.access_key) export AWS_ACCESS_KEY_ID @@ -70,6 +70,8 @@ if [[ "${USE_DRA_CREDENTIALS:-}" == "true" ]]; then export DRA_VAULT_ADDR fi +source .buildkite/scripts/third-party-test-credentials.sh + if [[ "${USE_SNYK_CREDENTIALS:-}" == "true" ]]; then SNYK_TOKEN=$(vault read -field=token secret/ci/elastic-elasticsearch/migrated/snyk) export SNYK_TOKEN diff --git a/.buildkite/pipelines/periodic.template.yml b/.buildkite/pipelines/periodic.template.yml index e4f844afc3f41..ec3ae76ffcdfb 100644 --- a/.buildkite/pipelines/periodic.template.yml +++ b/.buildkite/pipelines/periodic.template.yml @@ -103,6 +103,73 @@ steps: image: family/elasticsearch-ubuntu-2004 diskSizeGb: 350 machineType: custom-32-98304 + - group: third-party tests + steps: + - label: third-party / azure-sas + command: | + export azure_storage_container=elasticsearch-ci-thirdparty-sas + export azure_storage_base_path=$BUILDKITE_BRANCH + + .ci/scripts/run-gradle.sh azureThirdPartyTest + env: + USE_3RD_PARTY_AZURE_SAS_CREDENTIALS: "true" + timeout_in_minutes: 30 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: n2-standard-8 + buildDirectory: /dev/shm/bk + - label: third-party / azure + command: | + export azure_storage_container=elasticsearch-ci-thirdparty + export azure_storage_base_path=$BUILDKITE_BRANCH + + .ci/scripts/run-gradle.sh azureThirdPartyTest + env: + USE_3RD_PARTY_AZURE_CREDENTIALS: "true" + timeout_in_minutes: 30 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: n2-standard-8 + buildDirectory: /dev/shm/bk + - label: third-party / gcs + command: | + export google_storage_bucket=elasticsearch-ci-thirdparty + export google_storage_base_path=$BUILDKITE_BRANCH + + .ci/scripts/run-gradle.sh gcsThirdPartyTest + env: + USE_3RD_PARTY_GCS_CREDENTIALS: "true" + timeout_in_minutes: 30 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: n2-standard-8 + buildDirectory: /dev/shm/bk + - label: third-party / geoip + command: | + .ci/scripts/run-gradle.sh :modules:ingest-geoip:internalClusterTest -Dtests.jvm.argline="-Dgeoip_use_service=true" + timeout_in_minutes: 30 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: n2-standard-8 + buildDirectory: /dev/shm/bk + - label: third-party / s3 + command: | + export amazon_s3_bucket=elasticsearch-ci.us-west-2 + export amazon_s3_base_path=$BUILDKITE_BRANCH + + .ci/scripts/run-gradle.sh s3ThirdPartyTest + env: + USE_3RD_PARTY_S3_CREDENTIALS: "true" + timeout_in_minutes: 30 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: n2-standard-8 + buildDirectory: /dev/shm/bk - label: Upload Snyk Dependency Graph command: .ci/scripts/run-gradle.sh uploadSnykDependencyGraph -PsnykTargetReference=$BUILDKITE_BRANCH env: diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml index 301c3adf455c1..cfc7cfe5f01b1 100644 --- a/.buildkite/pipelines/periodic.yml +++ b/.buildkite/pipelines/periodic.yml @@ -1114,6 +1114,72 @@ steps: image: family/elasticsearch-ubuntu-2004 diskSizeGb: 350 machineType: custom-32-98304 + - group: third-party tests + steps: + - label: third-party / azure-sas + command: | + export azure_storage_container=elasticsearch-ci-thirdparty-sas + export azure_storage_base_path=$BUILDKITE_BRANCH + + .ci/scripts/run-gradle.sh azureThirdPartyTest + env: + USE_3RD_PARTY_AZURE_SAS_CREDENTIALS: "true" + timeout_in_minutes: 30 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: n2-standard-8 + buildDirectory: /dev/shm/bk + - label: third-party / azure + command: | + export azure_storage_container=elasticsearch-ci-thirdparty + export azure_storage_base_path=$BUILDKITE_BRANCH + + .ci/scripts/run-gradle.sh azureThirdPartyTest + env: + USE_3RD_PARTY_AZURE_CREDENTIALS: "true" + timeout_in_minutes: 30 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: n2-standard-8 + buildDirectory: /dev/shm/bk + - label: third-party / gcs + command: | + export google_storage_bucket=elasticsearch-ci-thirdparty + export google_storage_base_path=$BUILDKITE_BRANCH + + .ci/scripts/run-gradle.sh gcsThirdPartyTest + env: + USE_3RD_PARTY_GCS_CREDENTIALS: "true" + timeout_in_minutes: 30 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: n2-standard-8 + buildDirectory: /dev/shm/bk + - label: third-party / geoip + command: | + .ci/scripts/run-gradle.sh :modules:ingest-geoip:internalClusterTest -Dtests.jvm.argline="-Dgeoip_use_service=true" + timeout_in_minutes: 30 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: n2-standard-8 + - label: third-party / s3 + command: | + export amazon_s3_bucket=elasticsearch-ci.us-west-2 + export amazon_s3_base_path=$BUILDKITE_BRANCH + + .ci/scripts/run-gradle.sh s3ThirdPartyTest + env: + USE_3RD_PARTY_S3_CREDENTIALS: "true" + timeout_in_minutes: 30 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: n2-standard-8 + buildDirectory: /dev/shm/bk - label: Upload Snyk Dependency Graph command: .ci/scripts/run-gradle.sh uploadSnykDependencyGraph -PsnykTargetReference=$BUILDKITE_BRANCH env: diff --git a/.buildkite/scripts/get-legacy-secret.sh b/.buildkite/scripts/get-legacy-secret.sh new file mode 100755 index 0000000000000..3df6c27f484f2 --- /dev/null +++ b/.buildkite/scripts/get-legacy-secret.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -euo pipefail + +# WARNING: this script will echo the credentials to the console. It is meant to be called from another script and captured in a variable. +# It should really only be used inside .buildkite/hooks/pre-command + +source .buildkite/scripts/setup-legacy-vault.sh + +vault read -format=json "$1" diff --git a/.buildkite/scripts/lucene-snapshot/get-credentials.sh b/.buildkite/scripts/setup-legacy-vault.sh similarity index 61% rename from .buildkite/scripts/lucene-snapshot/get-credentials.sh rename to .buildkite/scripts/setup-legacy-vault.sh index 042c664384a7f..d84f2a94d5391 100755 --- a/.buildkite/scripts/lucene-snapshot/get-credentials.sh +++ b/.buildkite/scripts/setup-legacy-vault.sh @@ -2,9 +2,6 @@ set -euo pipefail -# WARNING: this script will echo the credentials to the console. It is meant to be called from another script and captured in a variable. -# It should really only be used inside .buildkite/hooks/pre-command - VAULT_ROLE_ID=$(vault read -field=role-id secret/ci/elastic-elasticsearch/legacy-vault-credentials) VAULT_SECRET_ID=$(vault read -field=secret-id secret/ci/elastic-elasticsearch/legacy-vault-credentials) VAULT_ADDR=https://secrets.elastic.co:8200 @@ -12,5 +9,3 @@ VAULT_ADDR=https://secrets.elastic.co:8200 unset VAULT_TOKEN VAULT_TOKEN=$(vault write -field=token auth/approle/login role_id=$VAULT_ROLE_ID secret_id=$VAULT_SECRET_ID) export VAULT_TOKEN - -vault read -format=json aws-elastic/creds/lucene-snapshots diff --git a/.buildkite/scripts/third-party-test-credentials.gcs.sh b/.buildkite/scripts/third-party-test-credentials.gcs.sh new file mode 100755 index 0000000000000..fd1b435ed484b --- /dev/null +++ b/.buildkite/scripts/third-party-test-credentials.gcs.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -euo pipefail + +# Usage: .buildkite/scripts/third-party-test-credentials.gcs.sh + +source .buildkite/scripts/setup-legacy-vault.sh + +vault read -field=private_key_data gcp-elastic-ci-prod/key/elasticsearch-ci-thirdparty-gcs | base64 --decode > "$1" diff --git a/.buildkite/scripts/third-party-test-credentials.sh b/.buildkite/scripts/third-party-test-credentials.sh new file mode 100755 index 0000000000000..c882d61cbade6 --- /dev/null +++ b/.buildkite/scripts/third-party-test-credentials.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +set -euo pipefail + +# You'll notice that most of the variables are exported twice with different names here +# The first/uppercase export is to ensure that Buildkite masks the values in the logs should they accidentally be output +# The second/lowercase export is what the tests expect/require + +if [[ "${USE_3RD_PARTY_AZURE_CREDENTIALS:-}" == "true" ]]; then + json=$(vault read -format=json secret/ci/elastic-elasticsearch/migrated/azure_thirdparty_test_creds) + + AZURE_STORAGE_ACCOUNT_SECRET=$(echo "$json" | jq -r .data.account_id) + export AZURE_STORAGE_ACCOUNT_SECRET + export azure_storage_account="$AZURE_STORAGE_ACCOUNT_SECRET" + + AZURE_STORAGE_KEY=$(echo "$json" | jq -r .data.account_key) + export AZURE_STORAGE_KEY + export azure_storage_key="$AZURE_STORAGE_KEY" +fi + +if [[ "${USE_3RD_PARTY_AZURE_SAS_CREDENTIALS:-}" == "true" ]]; then + json=$(vault read -format=json secret/ci/elastic-elasticsearch/migrated/azure_thirdparty_sas_test_creds) + + AZURE_STORAGE_ACCOUNT_SECRET=$(echo "$json" | jq -r .data.account_id) + export AZURE_STORAGE_ACCOUNT_SECRET + export azure_storage_account="$AZURE_STORAGE_ACCOUNT_SECRET" + + AZURE_STORAGE_SAS_TOKEN=$(echo "$json" | jq -r .data.account_sas_token) + export AZURE_STORAGE_SAS_TOKEN + export azure_storage_sas_token="$AZURE_STORAGE_SAS_TOKEN" +fi + +if [[ "${USE_3RD_PARTY_S3_CREDENTIALS:-}" == "true" ]]; then + json=$(.buildkite/scripts/get-legacy-secret.sh aws-test/creds/elasticsearch-ci-s3) + AMAZON_S3_ACCESS_KEY=$(echo "$json" | jq -r .data.access_key) + export AMAZON_S3_ACCESS_KEY + export amazon_s3_access_key="$AMAZON_S3_ACCESS_KEY" + + AMAZON_S3_SECRET_KEY=$(echo "$json" | jq -r .data.secret_key) + export AMAZON_S3_SECRET_KEY + export amazon_s3_secret_key="$AMAZON_S3_SECRET_KEY" +fi + +if [[ "${USE_3RD_PARTY_GCS_CREDENTIALS:-}" == "true" ]]; then + export google_storage_service_account=$(mktemp) + .buildkite/scripts/third-party-test-credentials.gcs.sh "$google_storage_service_account" +fi + + + +unset json diff --git a/test/fixtures/geoip-fixture/src/main/java/fixture/geoip/GeoIpHttpFixture.java b/test/fixtures/geoip-fixture/src/main/java/fixture/geoip/GeoIpHttpFixture.java index 6593bce0944fd..5cbfd750498f6 100644 --- a/test/fixtures/geoip-fixture/src/main/java/fixture/geoip/GeoIpHttpFixture.java +++ b/test/fixtures/geoip-fixture/src/main/java/fixture/geoip/GeoIpHttpFixture.java @@ -25,6 +25,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.StandardCopyOption; public class GeoIpHttpFixture extends ExternalResource { @@ -106,15 +107,25 @@ protected void after() { } private void copyFiles() throws Exception { - Files.copy(GeoIpHttpFixture.class.getResourceAsStream("/geoip-fixture/GeoLite2-ASN.tgz"), source.resolve("GeoLite2-ASN.tgz")); - Files.copy(GeoIpHttpFixture.class.getResourceAsStream("/geoip-fixture/GeoLite2-City.mmdb"), source.resolve("GeoLite2-City.mmdb")); + Files.copy( + GeoIpHttpFixture.class.getResourceAsStream("/geoip-fixture/GeoLite2-ASN.tgz"), + source.resolve("GeoLite2-ASN.tgz"), + StandardCopyOption.REPLACE_EXISTING + ); + Files.copy( + GeoIpHttpFixture.class.getResourceAsStream("/geoip-fixture/GeoLite2-City.mmdb"), + source.resolve("GeoLite2-City.mmdb"), + StandardCopyOption.REPLACE_EXISTING + ); Files.copy( GeoIpHttpFixture.class.getResourceAsStream("/geoip-fixture/GeoLite2-Country.mmdb"), - source.resolve("GeoLite2-Country.mmdb") + source.resolve("GeoLite2-Country.mmdb"), + StandardCopyOption.REPLACE_EXISTING ); Files.copy( GeoIpHttpFixture.class.getResourceAsStream("/geoip-fixture/MyCustomGeoLite2-City.mmdb"), - source.resolve("MyCustomGeoLite2-City.mmdb") + source.resolve("MyCustomGeoLite2-City.mmdb"), + StandardCopyOption.REPLACE_EXISTING ); new GeoIpCli().main(