Skip to content

Commit

Permalink
Merge branch '6.x' into ccr-6.x
Browse files Browse the repository at this point in the history
* 6.x: (96 commits)
  Introduce global checkpoint listeners (#32696)
  Use JDK 10 for 6.4 BWC builds (#32866)
  Remove unused imports - follow up to removal of test in issue 32855
  Removed flaky test. Looks like randomisation makes these assertions unreliable. This test is superfluous - it was added to address #32770 but it later turned out there was an existing test that just required a fix to provide the missing test coverage.
  [test] mute IndexShardTests.testDocStats
  Test: Fix forbidden uses in test framework (#32824)
  Security: remove password hash bootstrap check (#32440)
  Move validation to server for put user requests (#32471)
  [ML] Add high level REST client docs for ML put job endpoint (#32843)
  Painless: Change fqn_only to no_import (#32817)
  [test] mute testSearchWithSignificantTermsAgg
  Backport: CompletableContext class to avoid throwable (#32829)
  [TEST] Select free port for Minio (#32837)
  SCRIPTING: Support BucketAggScript return null (#32811) (#32833)
  HLRC: Add Delete License API (#32586)
  Aggregations/HL Rest client fix: missing scores (#32774)
  HLRC: migration get assistance API (#32744)
  Fix NOOP bulk updates (#32819)
  Increase logging testRetentionPolicyChangeDuringRecovery
  AwaitsFix case-functions.sql-spec
  ...
  • Loading branch information
jasontedor committed Aug 15, 2018
2 parents a418138 + 3b047ae commit 53cee21
Show file tree
Hide file tree
Showing 503 changed files with 14,196 additions and 5,558 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,8 @@ class BuildPlugin implements Plugin<Project> {
project.pluginManager.apply('nebula.info-scm')
project.pluginManager.apply('nebula.info-jar')

project.getTasks().create("buildResources", ExportElasticsearchBuildResourcesTask)

globalBuildInfo(project)
configureRepositories(project)
configureConfigurations(project)
Expand All @@ -101,6 +103,7 @@ class BuildPlugin implements Plugin<Project> {
configureDependenciesInfo(project)
}


/** Performs checks on the build environment and prints information about the build environment. */
static void globalBuildInfo(Project project) {
if (project.rootProject.ext.has('buildChecksDone') == false) {
Expand All @@ -116,12 +119,14 @@ class BuildPlugin implements Plugin<Project> {

final Map<Integer, String> javaVersions = [:]
for (int version = 7; version <= Integer.parseInt(minimumCompilerVersion.majorVersion); version++) {
javaVersions.put(version, findJavaHome(version));
if(System.getenv(getJavaHomeEnvVarName(version.toString())) != null) {
javaVersions.put(version, findJavaHome(version.toString()));
}
}

String javaVendor = System.getProperty('java.vendor')
String javaVersion = System.getProperty('java.version')
String gradleJavaVersionDetails = "${javaVendor} ${javaVersion}" +
String gradleJavaVersion = System.getProperty('java.version')
String gradleJavaVersionDetails = "${javaVendor} ${gradleJavaVersion}" +
" [${System.getProperty('java.vm.name')} ${System.getProperty('java.vm.version')}]"

String compilerJavaVersionDetails = gradleJavaVersionDetails
Expand All @@ -144,33 +149,33 @@ class BuildPlugin implements Plugin<Project> {
// Build debugging info
println '======================================='
println 'Elasticsearch Build Hamster says Hello!'
println '======================================='
println " Gradle Version : ${project.gradle.gradleVersion}"
println " OS Info : ${System.getProperty('os.name')} ${System.getProperty('os.version')} (${System.getProperty('os.arch')})"
if (gradleJavaVersionDetails != compilerJavaVersionDetails || gradleJavaVersionDetails != runtimeJavaVersionDetails) {
println " JDK Version (gradle) : ${gradleJavaVersionDetails}"
println " JAVA_HOME (gradle) : ${gradleJavaHome}"
println " JDK Version (compile) : ${compilerJavaVersionDetails}"
println " JAVA_HOME (compile) : ${compilerJavaHome}"
println " JDK Version (runtime) : ${runtimeJavaVersionDetails}"
println " JAVA_HOME (runtime) : ${runtimeJavaHome}"
println " Compiler JDK Version : ${getPaddedMajorVersion(compilerJavaVersionEnum)} (${compilerJavaVersionDetails})"
println " Compiler java.home : ${compilerJavaHome}"
println " Runtime JDK Version : ${getPaddedMajorVersion(runtimeJavaVersionEnum)} (${runtimeJavaVersionDetails})"
println " Runtime java.home : ${runtimeJavaHome}"
println " Gradle JDK Version : ${getPaddedMajorVersion(JavaVersion.toVersion(gradleJavaVersion))} (${gradleJavaVersionDetails})"
println " Gradle java.home : ${gradleJavaHome}"
} else {
println " JDK Version : ${gradleJavaVersionDetails}"
println " JDK Version : ${getPaddedMajorVersion(JavaVersion.toVersion(gradleJavaVersion))} (${gradleJavaVersionDetails})"
println " JAVA_HOME : ${gradleJavaHome}"
}
println " Random Testing Seed : ${project.testSeed}"
println '======================================='

// enforce Java version
if (compilerJavaVersionEnum < minimumCompilerVersion) {
final String message =
"the environment variable JAVA_HOME must be set to a JDK installation directory for Java ${minimumCompilerVersion}" +
"the compiler java.home must be set to a JDK installation directory for Java ${minimumCompilerVersion}" +
" but is [${compilerJavaHome}] corresponding to [${compilerJavaVersionEnum}]"
throw new GradleException(message)
}

if (runtimeJavaVersionEnum < minimumRuntimeVersion) {
final String message =
"the environment variable RUNTIME_JAVA_HOME must be set to a JDK installation directory for Java ${minimumRuntimeVersion}" +
"the runtime java.home must be set to a JDK installation directory for Java ${minimumRuntimeVersion}" +
" but is [${runtimeJavaHome}] corresponding to [${runtimeJavaVersionEnum}]"
throw new GradleException(message)
}
Expand Down Expand Up @@ -205,6 +210,7 @@ class BuildPlugin implements Plugin<Project> {
project.rootProject.ext.minimumCompilerVersion = minimumCompilerVersion
project.rootProject.ext.minimumRuntimeVersion = minimumRuntimeVersion
project.rootProject.ext.inFipsJvm = inFipsJvm
project.rootProject.ext.gradleJavaVersion = JavaVersion.toVersion(gradleJavaVersion)
}

project.targetCompatibility = project.rootProject.ext.minimumRuntimeVersion
Expand All @@ -217,11 +223,20 @@ class BuildPlugin implements Plugin<Project> {
project.ext.runtimeJavaVersion = project.rootProject.ext.runtimeJavaVersion
project.ext.javaVersions = project.rootProject.ext.javaVersions
project.ext.inFipsJvm = project.rootProject.ext.inFipsJvm
project.ext.gradleJavaVersion = project.rootProject.ext.gradleJavaVersion
}

private static String getPaddedMajorVersion(JavaVersion compilerJavaVersionEnum) {
compilerJavaVersionEnum.getMajorVersion().toString().padLeft(2)
}

private static String findCompilerJavaHome() {
final String javaHome = System.getenv('JAVA_HOME')
if (javaHome == null) {
final String compilerJavaHome = System.getenv('JAVA_HOME')
final String compilerJavaProperty = System.getProperty('compiler.java')
if (compilerJavaProperty != null) {
compilerJavaHome = findJavaHome(compilerJavaProperty)
}
if (compilerJavaHome == null) {
if (System.getProperty("idea.active") != null || System.getProperty("eclipse.launcher") != null) {
// IntelliJ does not set JAVA_HOME, so we use the JDK that Gradle was run with
return Jvm.current().javaHome
Expand All @@ -233,11 +248,24 @@ class BuildPlugin implements Plugin<Project> {
)
}
}
return javaHome
return compilerJavaHome
}

private static String findJavaHome(int version) {
return System.getenv('JAVA' + version + '_HOME')
private static String findJavaHome(String version) {
String versionedVarName = getJavaHomeEnvVarName(version)
String versionedJavaHome = System.getenv(versionedVarName);
if (versionedJavaHome == null) {
throw new GradleException(
"$versionedVarName must be set to build Elasticsearch. " +
"Note that if the variable was just set you might have to run `./gradlew --stop` for " +
"it to be picked up. See https://github.com/elastic/elasticsearch/issues/31399 details."
)
}
return versionedJavaHome
}

private static String getJavaHomeEnvVarName(String version) {
return 'JAVA' + version + '_HOME'
}

/** Add a check before gradle execution phase which ensures java home for the given java version is set. */
Expand Down Expand Up @@ -271,7 +299,10 @@ class BuildPlugin implements Plugin<Project> {
}

private static String findRuntimeJavaHome(final String compilerJavaHome) {
assert compilerJavaHome != null
String runtimeJavaProperty = System.getProperty("runtime.java")
if (runtimeJavaProperty != null) {
return findJavaHome(runtimeJavaProperty)
}
return System.getenv('RUNTIME_JAVA_HOME') ?: compilerJavaHome
}

Expand Down Expand Up @@ -769,6 +800,12 @@ class BuildPlugin implements Plugin<Project> {
systemProperty 'tests.security.manager', 'true'
systemProperty 'jna.nosys', 'true'
systemProperty 'es.scripting.exception_for_missing_value', 'true'
systemProperty 'compiler.java', project.ext.compilerJavaVersion.getMajorVersion()
if (project.ext.inFipsJvm) {
systemProperty 'runtime.java', project.ext.runtimeJavaVersion.getMajorVersion() + "FIPS"
} else {
systemProperty 'runtime.java', project.ext.runtimeJavaVersion.getMajorVersion()
}
// TODO: remove setting logging level via system property
systemProperty 'tests.logger.level', 'WARN'
for (Map.Entry<String, String> property : System.properties.entrySet()) {
Expand All @@ -783,9 +820,12 @@ class BuildPlugin implements Plugin<Project> {
}
}

// TODO: remove this once joda time is removed from scriptin in 7.0
// TODO: remove this once joda time is removed from scripting in 7.0
systemProperty 'es.scripting.use_java_time', 'true'

// TODO: remove this once ctx isn't added to update script params in 7.0
systemProperty 'es.scripting.update.ctx_in_params', 'false'

// Set the system keystore/truststore password if we're running tests in a FIPS-140 JVM
if (project.inFipsJvm) {
systemProperty 'javax.net.ssl.trustStorePassword', 'password'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -284,6 +284,10 @@ public class SnippetsTask extends DefaultTask {
contents.append(line).append('\n')
return
}
// Allow line continuations for console snippets within lists
if (snippet != null && line.trim() == '+') {
return
}
// Just finished
emit()
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.gradle;

import org.gradle.api.DefaultTask;
import org.gradle.api.GradleException;
import org.gradle.api.file.DirectoryProperty;
import org.gradle.api.file.RegularFileProperty;
import org.gradle.api.logging.Logger;
import org.gradle.api.logging.Logging;
import org.gradle.api.tasks.Classpath;
import org.gradle.api.tasks.Input;
import org.gradle.api.tasks.OutputDirectory;
import org.gradle.api.tasks.SkipWhenEmpty;
import org.gradle.api.tasks.StopExecutionException;
import org.gradle.api.tasks.TaskAction;

import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;

/**
* Export Elasticsearch build resources to configurable paths
* <p>
* Wil overwrite existing files and create missing directories.
* Useful for resources that that need to be passed to other processes trough the filesystem or otherwise can't be
* consumed from the classpath.
*/
public class ExportElasticsearchBuildResourcesTask extends DefaultTask {

private final Logger logger = Logging.getLogger(ExportElasticsearchBuildResourcesTask.class);

private final Set<String> resources = new HashSet<>();

private DirectoryProperty outputDir;

public ExportElasticsearchBuildResourcesTask() {
outputDir = getProject().getLayout().directoryProperty(
getProject().getLayout().getBuildDirectory().dir("build-tools-exported")
);
}

@OutputDirectory
public DirectoryProperty getOutputDir() {
return outputDir;
}

@Input
@SkipWhenEmpty
public Set<String> getResources() {
return Collections.unmodifiableSet(resources);
}

@Classpath
public String getResourcesClasspath() {
// This will make sure the task is not considered up to date if the resources are changed.
logger.info("Classpath: {}", System.getProperty("java.class.path"));
return System.getProperty("java.class.path");
}

public void setOutputDir(DirectoryProperty outputDir) {
this.outputDir = outputDir;
}

public RegularFileProperty take(String resource) {
if (getState().getExecuted() || getState().getExecuting()) {
throw new GradleException("buildResources can't be configured after the task ran. " +
"Make sure task is not used after configuration time"
);
}
resources.add(resource);
return getProject().getLayout().fileProperty(outputDir.file(resource));
}

@TaskAction
public void doExport() {
if (resources.isEmpty()) {
throw new StopExecutionException();
}
resources.stream().parallel()
.forEach(resourcePath -> {
Path destination = outputDir.get().file(resourcePath).getAsFile().toPath();
try (InputStream is = getClass().getClassLoader().getResourceAsStream(resourcePath)) {
if (is == null) {
throw new GradleException("Can't export `" + resourcePath + "` from build-tools: not found");
}
Files.copy(is, destination);
} catch (IOException e) {
throw new GradleException("Can't write resource `" + resourcePath + "` to " + destination);
}
});
}

}
1 change: 1 addition & 0 deletions buildSrc/src/main/resources/checkstyle_suppressions.xml
Original file line number Diff line number Diff line change
Expand Up @@ -720,6 +720,7 @@
<suppress files="modules[/\\]lang-expression[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]expression[/\\]ExpressionScriptEngine.java" checks="LineLength" />
<suppress files="modules[/\\]lang-expression[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]expression[/\\]MoreExpressionTests.java" checks="LineLength" />
<suppress files="modules[/\\]lang-expression[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]expression[/\\]StoredExpressionTests.java" checks="LineLength" />
<suppress files="modules[/\\]lang-painless[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]painless[/\\]ContextExampleTests.java" checks="LineLength" />
<suppress files="modules[/\\]reindex[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]reindex[/\\]TransportUpdateByQueryAction.java" checks="LineLength" />
<suppress files="plugins[/\\]analysis-icu[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]IcuCollationTokenFilterFactory.java" checks="LineLength" />
<suppress files="plugins[/\\]analysis-icu[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]IcuFoldingTokenFilterFactory.java" checks="LineLength" />
Expand Down
Loading

0 comments on commit 53cee21

Please sign in to comment.