-
Notifications
You must be signed in to change notification settings - Fork 8k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[CI] Buildkite support with Baseline pipeline
- Loading branch information
1 parent
ebf77da
commit 5eb72ea
Showing
33 changed files
with
715 additions
and
3 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,76 @@ | ||
{ | ||
"gcp": { | ||
"project": "elastic-kibana-ci", | ||
"zones": ["us-central1-a", "us-central1-b", "us-central1-c", "us-central1-f"], | ||
"serviceAccount": "elastic-buildkite-agent@elastic-kibana-ci.iam.gserviceaccount.com", | ||
"imageFamily": "kb-ubuntu", | ||
"diskType": "pd-ssd", | ||
"diskSizeGb": 75, | ||
"overprovision": 0, | ||
"minimumAgents": 0, | ||
"maximumAgents": 50, | ||
"gracefulStopAfterMins": 360, | ||
"hardStopAfterMins": 540, | ||
"idleTimeoutMins": 10, | ||
"exitAfterOneJob": false, | ||
|
||
"agents": [ | ||
{ | ||
"queue": "default", | ||
"name": "kb-default", | ||
"minimumAgents": 1, | ||
"maximumAgents": 100, | ||
"idleTimeoutMins": 60, | ||
"machineType": "e2-small" | ||
}, | ||
{ | ||
"queue": "baseline", | ||
"name": "kb-baseline", | ||
"machineType": "c2-standard-8", | ||
"diskSizeGb": 128 | ||
}, | ||
{ | ||
"queue": "baseline-small", | ||
"name": "kb-baseline-small", | ||
"machineType": "c2-standard-4" | ||
}, | ||
{ | ||
"queue": "jest", | ||
"name": "kb-jest", | ||
"machineType": "n2-standard-2", | ||
"diskSizeGb": 128 | ||
}, | ||
{ | ||
"queue": "ci-group", | ||
"name": "kb-cigroup", | ||
"machineType": "n2-standard-8", | ||
"diskSizeGb": 256 | ||
}, | ||
{ | ||
"queue": "ci-group-4", | ||
"name": "kb-cigroup-4", | ||
"machineType": "n2-standard-4", | ||
"diskSizeGb": 128 | ||
}, | ||
{ | ||
"queue": "ci-group-4d", | ||
"name": "kb-cigroup-4d", | ||
"machineType": "n2d-standard-4", | ||
"diskSizeGb": 128 | ||
}, | ||
{ | ||
"queue": "ci-group-6", | ||
"name": "kb-cigroup-6", | ||
"machineType": "n2-custom-6-16384", | ||
"diskSizeGb": 128 | ||
}, | ||
{ | ||
"queue": "packer", | ||
"name": "kb-packer", | ||
"serviceAccount": "buildkite-packer-agent@elastic-kibana-184716.iam.gserviceaccount.com", | ||
"maximumAgents": 10, | ||
"machineType": "e2-small" | ||
} | ||
] | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,29 @@ | ||
env: | ||
GITHUB_COMMIT_STATUS_ENABLED: 'true' | ||
GITHUB_COMMIT_STATUS_CONTEXT: 'buildkite/kibana-ci-baseline' | ||
steps: | ||
- command: .buildkite/setup/pre_build.sh | ||
label: Pre-Build | ||
|
||
- wait | ||
|
||
- command: .buildkite/scripts/jobs/baseline.sh | ||
label: Default Build and Metrics | ||
agents: | ||
queue: baseline | ||
|
||
- command: .buildkite/scripts/jobs/baseline_ts_refs.sh | ||
label: Build TS Refs | ||
agents: | ||
queue: baseline-small | ||
|
||
- command: .buildkite/scripts/jobs/baseline_public_api_docs.sh | ||
label: Check Public API Docs | ||
agents: | ||
queue: baseline-small | ||
|
||
- wait: ~ | ||
continue_on_failure: true | ||
|
||
- command: .buildkite/setup/post_build.sh | ||
label: Post-Build |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,7 @@ | ||
#!/usr/bin/env bash | ||
|
||
set -euo pipefail | ||
|
||
if [[ "$BUILDKITE_COMMAND_EXIT_STATUS" != "0" ]]; then | ||
buildkite-agent meta-data set build_failed true | ||
fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,56 @@ | ||
#!/usr/bin/env bash | ||
|
||
set -euo pipefail | ||
|
||
CI_STATS_BUILD_ID="$(buildkite-agent meta-data get ci_stats_build_id --default '')" | ||
export CI_STATS_BUILD_ID | ||
|
||
if [[ "${ES_SNAPSHOT_MANIFEST:-}" ]]; then | ||
cat << EOF | buildkite-agent annotate --style "info" --context es-snapshot-manifest | ||
This build is running using a custom Elasticsearch snapshot. | ||
ES Snapshot Manifest: $ES_SNAPSHOT_MANIFEST | ||
To use this locally, simply prefix your commands with: | ||
\`\`\` | ||
ES_SNAPSHOT_MANIFEST="$ES_SNAPSHOT_MANIFEST" | ||
\`\`\` | ||
e.g. | ||
\`\`\` | ||
ES_SNAPSHOT_MANIFEST="$ES_SNAPSHOT_MANIFEST" node scripts/functional_tests_server.js | ||
\`\`\` | ||
EOF | ||
fi | ||
|
||
ES_SNAPSHOT_MANIFEST=${ES_SNAPSHOT_MANIFEST:-$(buildkite-agent meta-data get ES_SNAPSHOT_MANIFEST --default '')} | ||
export ES_SNAPSHOT_MANIFEST | ||
|
||
if [[ "$CI_STATS_BUILD_ID" ]]; then | ||
echo "CI Stats Build ID: $CI_STATS_BUILD_ID" | ||
|
||
CI_STATS_TOKEN="$(vault read -field=api_token secret/kibana-issues/dev/kibana_ci_stats)" | ||
export CI_STATS_TOKEN | ||
|
||
CI_STATS_HOST="$(vault read -field=api_host secret/kibana-issues/dev/kibana_ci_stats)" | ||
export CI_STATS_HOST | ||
|
||
KIBANA_CI_STATS_CONFIG=$(jq -n \ | ||
--arg buildId "$CI_STATS_BUILD_ID" \ | ||
--arg apiUrl "https://$CI_STATS_HOST" \ | ||
--arg apiToken "$CI_STATS_TOKEN" \ | ||
'{buildId: $buildId, apiUrl: $apiUrl, apiToken: $apiToken}' \ | ||
) | ||
export KIBANA_CI_STATS_CONFIG | ||
fi | ||
|
||
GITHUB_TOKEN=$(vault read -field=github_token secret/kibana-issues/dev/kibanamachine) | ||
export GITHUB_TOKEN | ||
|
||
if [[ "${SKIP_CI_SETUP:-}" != "true" ]]; then | ||
if [[ -d .buildkite/scripts && "${BUILDKITE_COMMAND:-}" != "buildkite-agent pipeline upload"* ]]; then | ||
source .buildkite/scripts/setup.sh | ||
fi | ||
fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,24 @@ | ||
#!/usr/bin/env bash | ||
|
||
set -euo pipefail | ||
|
||
source "$(dirname "${0}")/util.sh" | ||
|
||
if [[ ! -f "target/.bootstrap-cache" ]]; then | ||
echo "--- yarn install and bootstrap" | ||
yarn kbn bootstrap --verbose | ||
|
||
### | ||
### upload ts-refs-cache artifacts as quickly as possible so they are available for download | ||
### | ||
if [[ "${BUILD_TS_REFS_CACHE_CAPTURE:-}" == "true" ]]; then | ||
echo "--- Upload ts-refs-cache" | ||
cd "$KIBANA_DIR/target/ts_refs_cache" | ||
gsutil cp "*.zip" 'gs://kibana-ci-ts-refs-cache/' | ||
cd "$KIBANA_DIR" | ||
fi | ||
|
||
if [[ "$DISABLE_BOOTSTRAP_VALIDATION" != "true" ]]; then | ||
verify_no_git_changes 'yarn kbn bootstrap' | ||
fi | ||
fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,16 @@ | ||
#!/usr/bin/env bash | ||
|
||
set -euo pipefail | ||
|
||
export KBN_NP_PLUGINS_BUILT=true | ||
|
||
echo "--- Build Kibana Distribution" | ||
node scripts/build --debug --no-oss | ||
|
||
echo "--- Archive Kibana Distribution" | ||
linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')" | ||
installDir="$KIBANA_DIR/install/kibana" | ||
mkdir -p "$installDir" | ||
tar -xzf "$linuxBuild" -C "$installDir" --strip=1 | ||
mkdir -p "$KIBANA_BUILD_LOCATION" | ||
cp -pR install/kibana/. "$KIBANA_BUILD_LOCATION/" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,32 @@ | ||
#!/usr/bin/env bash | ||
|
||
set -euo pipefail | ||
|
||
echo "--- Build Platform Plugins" | ||
node scripts/build_kibana_platform_plugins \ | ||
--scan-dir "$KIBANA_DIR/test/plugin_functional/plugins" \ | ||
--scan-dir "$KIBANA_DIR/test/interpreter_functional/plugins" \ | ||
--scan-dir "$KIBANA_DIR/test/common/fixtures/plugins" \ | ||
--scan-dir "$KIBANA_DIR/examples" \ | ||
--scan-dir "$KIBANA_DIR/test/plugin_functional/plugins" \ | ||
--scan-dir "$KIBANA_DIR/test/common/fixtures/plugins" \ | ||
--scan-dir "$XPACK_DIR/test/plugin_functional/plugins" \ | ||
--scan-dir "$XPACK_DIR/test/functional_with_es_ssl/fixtures/plugins" \ | ||
--scan-dir "$XPACK_DIR/test/alerting_api_integration/plugins" \ | ||
--scan-dir "$XPACK_DIR/test/plugin_api_integration/plugins" \ | ||
--scan-dir "$XPACK_DIR/test/plugin_api_perf/plugins" \ | ||
--scan-dir "$XPACK_DIR/test/licensing_plugin/plugins" \ | ||
--scan-dir "$XPACK_DIR/test/usage_collection/plugins" \ | ||
--scan-dir "$XPACK_DIR/test/security_functional/fixtures/common" \ | ||
--scan-dir "$XPACK_DIR/examples" \ | ||
--verbose | ||
|
||
echo "--- Archive built plugins" | ||
shopt -s globstar | ||
tar -zcf \ | ||
target/kibana-default-plugins.tar.gz \ | ||
x-pack/plugins/**/target/public \ | ||
x-pack/test/**/target/public \ | ||
examples/**/target/public \ | ||
x-pack/examples/**/target/public \ | ||
test/**/target/public |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,19 @@ | ||
#!/usr/bin/env bash | ||
|
||
set -euo pipefail | ||
|
||
if [[ ! -d "$KIBANA_BUILD_LOCATION/bin" ]]; then | ||
echo '--- Downloading Distribution and Plugin artifacts' | ||
|
||
cd "$WORKSPACE" | ||
|
||
buildkite-agent artifact download kibana-default.tar.gz . | ||
buildkite-agent artifact download kibana-default-plugins.tar.gz . | ||
|
||
mkdir -p "$KIBANA_BUILD_LOCATION" | ||
tar -xzf kibana-default.tar.gz -C "$KIBANA_BUILD_LOCATION" --strip=1 | ||
|
||
cd "$KIBANA_DIR" | ||
|
||
tar -xzf ../kibana-default-plugins.tar.gz | ||
fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,76 @@ | ||
#!/usr/bin/env bash | ||
|
||
set -euo pipefail | ||
|
||
source .buildkite/scripts/util.sh | ||
|
||
export CI=true | ||
|
||
KIBANA_DIR=$(pwd) | ||
export KIBANA_DIR | ||
export XPACK_DIR="$KIBANA_DIR/x-pack" | ||
|
||
export CACHE_DIR="$HOME/.kibana" | ||
PARENT_DIR="$(cd "$KIBANA_DIR/.."; pwd)" | ||
export PARENT_DIR | ||
export WORKSPACE="${WORKSPACE:-$PARENT_DIR}" | ||
|
||
KIBANA_PKG_BRANCH="$(jq -r .branch "$KIBANA_DIR/package.json")" | ||
export KIBANA_PKG_BRANCH | ||
export KIBANA_BASE_BRANCH="$KIBANA_PKG_BRANCH" | ||
|
||
export GECKODRIVER_CDNURL="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache" | ||
export CHROMEDRIVER_CDNURL="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache" | ||
export RE2_DOWNLOAD_MIRROR="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache" | ||
export CYPRESS_DOWNLOAD_MIRROR="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache/cypress" | ||
|
||
export NODE_OPTIONS="--max-old-space-size=4096" | ||
|
||
export FORCE_COLOR=1 | ||
export TEST_BROWSER_HEADLESS=1 | ||
|
||
export ELASTIC_APM_ENVIRONMENT=ci | ||
export ELASTIC_APM_TRANSACTION_SAMPLE_RATE=0.1 | ||
|
||
CI_REPORTING_ENABLED=false # TODO enable when ready, only controls checks reporter and APM | ||
|
||
if is_pr; then | ||
export ELASTIC_APM_ACTIVE=false | ||
export CHECKS_REPORTER_ACTIVE="${CI_REPORTING_ENABLED-}" | ||
|
||
# These can be removed once we're not supporting Jenkins and Buildkite at the same time | ||
# These are primarily used by github checks reporter and can be configured via /github_checks_api.json | ||
export ghprbGhRepository="elastic/kibana" | ||
export ghprbActualCommit="$BUILDKITE_COMMIT" | ||
export BUILD_URL="$BUILDKITE_BUILD_URL" | ||
|
||
# set_git_merge_base # TODO for PRs | ||
else | ||
export ELASTIC_APM_ACTIVE="${CI_REPORTING_ENABLED-}" | ||
export CHECKS_REPORTER_ACTIVE=false | ||
fi | ||
|
||
export FLEET_PACKAGE_REGISTRY_PORT=6104 | ||
export TEST_CORS_SERVER_PORT=6105 | ||
|
||
export DETECT_CHROMEDRIVER_VERSION=true | ||
export CHROMEDRIVER_FORCE_DOWNLOAD=true | ||
|
||
export GCS_UPLOAD_PREFIX=FAKE_UPLOAD_PREFIX # TODO remove the need for this | ||
|
||
export KIBANA_BUILD_LOCATION="$WORKSPACE/kibana-build-xpack" | ||
|
||
if [[ "${BUILD_TS_REFS_CACHE_ENABLE:-}" != "true" ]]; then | ||
export BUILD_TS_REFS_CACHE_ENABLE=false | ||
fi | ||
|
||
export BUILD_TS_REFS_DISABLE=true | ||
export DISABLE_BOOTSTRAP_VALIDATION=true | ||
|
||
export TEST_KIBANA_HOST=localhost | ||
export TEST_KIBANA_PORT=6101 | ||
export TEST_KIBANA_URL="http://elastic:changeme@localhost:6101" | ||
export TEST_ES_URL="http://elastic:changeme@localhost:6102" | ||
export TEST_ES_TRANSPORT_PORT=6103 | ||
export TEST_CORS_SERVER_PORT=6106 | ||
export ALERTING_PROXY_PORT=6105 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,11 @@ | ||
#!/usr/bin/env bash | ||
|
||
set -euo pipefail | ||
|
||
export DISABLE_BOOTSTRAP_VALIDATION=true | ||
export BUILD_TS_REFS_DISABLE=true | ||
|
||
.buildkite/scripts/bootstrap.sh | ||
.buildkite/scripts/build_kibana.sh | ||
.buildkite/scripts/post_build_kibana.sh | ||
.buildkite/scripts/saved_object_field_metrics.sh |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,8 @@ | ||
#!/usr/bin/env bash | ||
|
||
set -euo pipefail | ||
|
||
.buildkite/scripts/bootstrap.sh | ||
|
||
echo "--- Build API Docs" | ||
node scripts/build_api_docs |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,10 @@ | ||
#!/usr/bin/env bash | ||
|
||
set -euo pipefail | ||
|
||
export BUILD_TS_REFS_CACHE_ENABLE=true | ||
export BUILD_TS_REFS_CACHE_CAPTURE=true | ||
export DISABLE_BOOTSTRAP_VALIDATION=true | ||
export BUILD_TS_REFS_DISABLE=false | ||
|
||
.buildkite/scripts/bootstrap.sh |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,8 @@ | ||
#!/usr/bin/env bash | ||
|
||
set -euo pipefail | ||
|
||
source .buildkite/scripts/env.sh | ||
source .buildkite/scripts/setup_node.sh | ||
|
||
yarn install |
Oops, something went wrong.