forked from elastic/elasticsearch
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'master' into unbound_id_response_timeout
- Loading branch information
Showing
5,480 changed files
with
199,283 additions
and
103,165 deletions.
The diff you're trying to view is too large. We only load the first 3000 changed files.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,16 @@ | ||
CI configuration for Elasticsearch | ||
================================== | ||
|
||
Layout and Syntax | ||
----------------- | ||
|
||
CI is run by Jenkins at [elasticsearch-ci](https://elasticsearch-ci.elastic.co/). | ||
Jobs live in the [jobs.t](jobs.t) directory, these are defined in YML using a syntax | ||
simmilar to [JJB](https://elasticsearch-ci.elastic.co/view/Elasticsearch%20master/). | ||
Macros are not allowed, and each job must be defined in its own file. | ||
Merging of the default configuration is customized so unlike in standard JJB, | ||
it recurses into YML objects. | ||
Further (internal) documentation on the setup | ||
[is available](https://github.com/elastic/infra/blob/master/flavortown/jjbb/README.md) | ||
. | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,32 @@ | ||
#!/bin/bash | ||
|
||
# drop page cache and kernel slab objects on linux | ||
[[ -x /usr/local/sbin/drop-caches ]] && sudo /usr/local/sbin/drop-caches | ||
|
||
rm -Rfv ~/.gradle/init.d/init.gradle | ||
mkdir -p ~/.gradle/init.d && cp -v $WORKSPACE/.ci/init.gradle ~/.gradle/init.d | ||
|
||
if [ -f /proc/cpuinfo ] ; then | ||
MAX_WORKERS=`grep '^cpu\scores' /proc/cpuinfo | uniq | sed 's/\s\+//g' | cut -d':' -f 2` | ||
else | ||
if [[ "$OSTYPE" == "darwin"* ]]; then | ||
# Parallel is disabled at this time (eventually set to 1) due to errors on the Mac workers | ||
# We'll have to do more testing to see if this can be re-enabled or what the proper value is. | ||
# MAX_WORKERS=`sysctl -n hw.physicalcpu | sed 's/\s\+//g'` | ||
MAX_WORKERS=2 | ||
else | ||
echo "Unsupported OS Type: $OSTYPE" | ||
exit 1 | ||
fi | ||
fi | ||
|
||
if pwd | grep -v -q ^/dev/shm ; then | ||
echo "Not running on a ramdisk, reducing number of workers" | ||
MAX_WORKERS=$(($MAX_WORKERS*2/3)) | ||
fi | ||
|
||
set -e | ||
./gradlew --parallel --scan \ | ||
-Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ \ | ||
--parallel --max-workers=$MAX_WORKERS \ | ||
"$@" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,12 @@ | ||
# This file specifies the Linux OS versions on which we can't build and | ||
# test Docker images for some reason. These values correspond to ID and | ||
# VERSION_ID from /etc/os-release, and a matching value will cause the | ||
# Docker tests to be skipped on that OS. If /etc/os-release doesn't exist | ||
# (as is the case on centos-6, for example) then that OS will again be | ||
# excluded. | ||
centos-6 | ||
debian-8 | ||
opensuse-15-1 | ||
ol-6.10 | ||
ol-7.7 | ||
sles-12 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,114 +1,114 @@ | ||
import com.bettercloud.vault.VaultConfig; | ||
import com.bettercloud.vault.Vault; | ||
import com.bettercloud.vault.VaultConfig | ||
import com.bettercloud.vault.Vault | ||
|
||
initscript { | ||
repositories { | ||
mavenCentral() | ||
} | ||
dependencies { | ||
classpath 'com.bettercloud:vault-java-driver:4.1.0' | ||
} | ||
repositories { | ||
mavenCentral() | ||
} | ||
dependencies { | ||
classpath 'com.bettercloud:vault-java-driver:4.1.0' | ||
} | ||
} | ||
|
||
boolean USE_ARTIFACTORY=false | ||
boolean USE_ARTIFACTORY = false | ||
|
||
if (System.getenv('VAULT_ADDR') == null) { | ||
throw new GradleException("You must set the VAULT_ADDR environment variable to use this init script.") | ||
throw new GradleException("You must set the VAULT_ADDR environment variable to use this init script.") | ||
} | ||
|
||
if (System.getenv('VAULT_ROLE_ID') == null && System.getenv('VAULT_SECRET_ID') == null && System.getenv('VAULT_TOKEN') == null) { | ||
throw new GradleException("You must set either the VAULT_ROLE_ID and VAULT_SECRET_ID environment variables, " + | ||
"or the VAULT_TOKEN environment variable to use this init script.") | ||
throw new GradleException("You must set either the VAULT_ROLE_ID and VAULT_SECRET_ID environment variables, " + | ||
"or the VAULT_TOKEN environment variable to use this init script.") | ||
} | ||
|
||
final String vaultToken = System.getenv('VAULT_TOKEN') ?: new Vault( | ||
new VaultConfig() | ||
.address(System.env.VAULT_ADDR) | ||
.engineVersion(1) | ||
.build() | ||
) | ||
.withRetries(5, 1000) | ||
.auth() | ||
.loginByAppRole("approle", System.env.VAULT_ROLE_ID, System.env.VAULT_SECRET_ID) | ||
.getAuthClientToken(); | ||
new VaultConfig() | ||
.address(System.env.VAULT_ADDR) | ||
.engineVersion(1) | ||
.build() | ||
) | ||
.withRetries(5, 1000) | ||
.auth() | ||
.loginByAppRole("approle", System.env.VAULT_ROLE_ID, System.env.VAULT_SECRET_ID) | ||
.getAuthClientToken() | ||
|
||
final Vault vault = new Vault( | ||
new VaultConfig() | ||
.address(System.env.VAULT_ADDR) | ||
.engineVersion(1) | ||
.token(vaultToken) | ||
.build() | ||
new VaultConfig() | ||
.address(System.env.VAULT_ADDR) | ||
.engineVersion(1) | ||
.token(vaultToken) | ||
.build() | ||
) | ||
.withRetries(5, 1000) | ||
.withRetries(5, 1000) | ||
|
||
|
||
if (USE_ARTIFACTORY) { | ||
final Map<String,String> artifactoryCredentials = vault.logical() | ||
.read("secret/elasticsearch-ci/artifactory.elstc.co") | ||
.getData(); | ||
logger.info("Using elastic artifactory repos") | ||
Closure configCache = { | ||
return { | ||
name "artifactory-gradle-release" | ||
url "https://artifactory.elstc.co/artifactory/gradle-release" | ||
credentials { | ||
username artifactoryCredentials.get("username") | ||
password artifactoryCredentials.get("token") | ||
} | ||
} | ||
final Map<String, String> artifactoryCredentials = vault.logical() | ||
.read("secret/elasticsearch-ci/artifactory.elstc.co") | ||
.getData() | ||
logger.info("Using elastic artifactory repos") | ||
Closure configCache = { | ||
return { | ||
name "artifactory-gradle-release" | ||
url "https://artifactory.elstc.co/artifactory/gradle-release" | ||
credentials { | ||
username artifactoryCredentials.get("username") | ||
password artifactoryCredentials.get("token") | ||
} | ||
} | ||
settingsEvaluated { settings -> | ||
settings.pluginManagement { | ||
repositories { | ||
maven configCache() | ||
} | ||
} | ||
} | ||
settingsEvaluated { settings -> | ||
settings.pluginManagement { | ||
repositories { | ||
maven configCache() | ||
} | ||
} | ||
projectsLoaded { | ||
allprojects { | ||
buildscript { | ||
repositories { | ||
maven configCache() | ||
} | ||
} | ||
repositories { | ||
maven configCache() | ||
} | ||
} | ||
projectsLoaded { | ||
allprojects { | ||
buildscript { | ||
repositories { | ||
maven configCache() | ||
} | ||
} | ||
repositories { | ||
maven configCache() | ||
} | ||
} | ||
} | ||
} | ||
|
||
projectsLoaded { | ||
rootProject { | ||
project.pluginManager.withPlugin('com.gradle.build-scan') { | ||
buildScan.server = 'https://gradle-enterprise.elastic.co' | ||
} | ||
rootProject { | ||
project.pluginManager.withPlugin('com.gradle.build-scan') { | ||
buildScan.server = 'https://gradle-enterprise.elastic.co' | ||
} | ||
} | ||
} | ||
|
||
|
||
final String buildCacheUrl = System.getProperty('org.elasticsearch.build.cache.url') | ||
final boolean buildCachePush = Boolean.valueOf(System.getProperty('org.elasticsearch.build.cache.push', 'false')) | ||
|
||
if (buildCacheUrl) { | ||
final Map<String,String> buildCacheCredentials = vault.logical() | ||
.read("secret/elasticsearch-ci/gradle-build-cache") | ||
.getData(); | ||
gradle.settingsEvaluated { settings -> | ||
settings.buildCache { | ||
local { | ||
// Disable the local build cache in CI since we use ephemeral workers and it incurs an IO penalty | ||
enabled = false | ||
} | ||
remote(HttpBuildCache) { | ||
url = buildCacheUrl | ||
push = buildCachePush | ||
credentials { | ||
username = buildCacheCredentials.get("username") | ||
password = buildCacheCredentials.get("password") | ||
} | ||
} | ||
final Map<String, String> buildCacheCredentials = vault.logical() | ||
.read("secret/elasticsearch-ci/gradle-build-cache") | ||
.getData() | ||
gradle.settingsEvaluated { settings -> | ||
settings.buildCache { | ||
local { | ||
// Disable the local build cache in CI since we use ephemeral workers and it incurs an IO penalty | ||
enabled = false | ||
} | ||
remote(HttpBuildCache) { | ||
url = buildCacheUrl | ||
push = buildCachePush | ||
credentials { | ||
username = buildCacheCredentials.get("username") | ||
password = buildCacheCredentials.get("password") | ||
} | ||
} | ||
} | ||
} | ||
} | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,93 @@ | ||
--- | ||
|
||
##### GLOBAL METADATA | ||
|
||
- meta: | ||
cluster: elasticsearch-ci | ||
|
||
##### JOB DEFAULTS | ||
|
||
- job: | ||
vault: | ||
url: https://secrets.elastic.co:8200 | ||
role_id: 1ba1ac3e-aee4-d040-d9a3-6ae23bd2b3db | ||
node: "general-purpose" | ||
concurrent: true | ||
logrotate: | ||
daysToKeep: 30 | ||
numToKeep: 90 | ||
artifactDaysToKeep: 7 | ||
scm: | ||
- git: | ||
name: origin | ||
# master node jenkins user ~/.ssh | ||
credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba | ||
reference-repo: "/var/lib/jenkins/.git-references/elasticsearch.git" | ||
branches: | ||
- "%BRANCH%" | ||
url: "https://github.com/elastic/elasticsearch.git" | ||
basedir: "" | ||
wipe-workspace: "yes" | ||
wrappers: | ||
- timeout: | ||
type: absolute | ||
timeout: "120" | ||
fail: true | ||
- ansicolor | ||
- timestamps | ||
- gradle-build-scan | ||
- inject: | ||
properties-file: '.ci/java-versions.properties' | ||
properties-content: | | ||
WORKSPACE=/dev/shm/$JOB_NAME | ||
JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA | ||
RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA | ||
JAVA7_HOME=$HOME/.java/java7 | ||
JAVA8_HOME=$HOME/.java/java8 | ||
JAVA9_HOME=$HOME/.java/java9 | ||
JAVA10_HOME=$HOME/.java/java10 | ||
JAVA11_HOME=$HOME/.java/java11 | ||
JAVA12_HOME=$HOME/.java/openjdk12 | ||
JAVA13_HOME=$HOME/.java/openjdk13 | ||
GRADLE_OPTS=-XX:+HeapDumpOnOutOfMemoryError -Xmx128m -Xms128m | ||
properties: | ||
- github: | ||
url: https://github.com/elastic/elasticsearch/ | ||
- inject: | ||
properties-content: | | ||
HOME=$JENKINS_HOME | ||
builders: | ||
- shell: | | ||
#!/usr/local/bin/runbld --redirect-stderr --last-good-commit elastic+elasticsearch+%BRANCH%+git+push | ||
.ci/build.sh -Dbwc.checkout.align=true check | ||
publishers: | ||
- email: | ||
recipients: infra-root+build@elastic.co | ||
# Upload additional logs | ||
- google-cloud-storage: | ||
credentials-id: 'elasticsearch-ci-gcs-plugin' | ||
uploads: | ||
- classic: | ||
file-pattern: 'build/*.tar.bz2' | ||
storage-location: 'gs://elasticsearch-ci-artifacts/jobs/$JOB_NAME' | ||
share-publicly: false | ||
upload-for-failed-jobs: true | ||
show-inline: true | ||
# Notify homer | ||
- postbuildscript: | ||
builders: | ||
- role: SLAVE | ||
build-on: | ||
- SUCCESS | ||
- FAILURE | ||
- UNSTABLE | ||
build-steps: | ||
- shell: | | ||
curl -sS -X POST \ | ||
-H "Content-Type: text/plain" \ | ||
--data "$BUILD_URL" \ | ||
"https://homer.app.elstc.co/webhook/jenkins/build-finished" || true | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,6 @@ | ||
- job: | ||
name: elastic+elasticsearch+%BRANCH%+periodic-next | ||
workspace: /dev/shm/elastic+elasticsearch+%BRANCH%+periodic | ||
display-name: "elastic / elasticsearch # %BRANCH% - periodic (experimental)" | ||
description: "Periodic testing of the Elasticsearch %BRANCH% branch.\n" | ||
triggers: [] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,11 @@ | ||
#!/bin/bash | ||
|
||
if [ -z "$BRANCH" ] ; then | ||
echo "BRANCH is unset" | ||
exit 1 | ||
fi | ||
|
||
rm -Rf .ci/jobs | ||
cp -r .ci/jobs.t .ci/jobs | ||
|
||
sed -i "s/%BRANCH%/${BRANCH}/g" .ci/jobs/*.yml |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -6,4 +6,4 @@ | |
# or 'openjdk' followed by the major release number. | ||
|
||
ES_BUILD_JAVA: | ||
- openjdk12 | ||
- openjdk13 |
Oops, something went wrong.