Skip to content

Commit

Permalink
Merge branch '6.x' into ccr-6.x
Browse files Browse the repository at this point in the history 
* 6.x:
  Docs: Remove duplicate test setup
  Docs: Fix description of percentile ranks example example (#31652)
  Remove deprecation warnings to prepare for Gradle 5 (sourceSets.main.output.classesDirs) (#30389)
  Do not check for Azure container existence (#31617)
  Merge AwsS3Service and InternalAwsS3Service in a S3Service class (#31580)
  Remove item listed in 6.3 notes that's not in 6.3 (#31623)
  remove unused import
  Upgrade gradle wrapper to 4.8 (#31525)
  Only set vm.max_map_count if greater than default (#31512)
  QA: Merge query-builder-bwc to restart test (#30979)
  Docs: Skip xpack snippet tests if no xpack (#31619)
  [TEST] Fix RemoteClusterConnectionTests
  Remove legacy MetaDataStateFormat (#31603)
  [TEST] call yaml client close method from test suite (#31591)
  [TEST] Close additional clients created while running yaml tests (#31575)
  Node selector per client rather than per request (#31471)
  Preserve thread context when connecting to remote cluster (#31574)
  Remove redundant 'minimum_should_match'
  Unify headers for full text queries
  JDBC driver prepared statement set* methods  (#31494)
  add logging breaking changes from 5.3 to 6.0 (#31583)
  Fix a formatting issue in the docvalue_fields documentation. (#31563)
  Improve robustness of geo shape parser for malformed shapes
  QA: Create xpack yaml features (#31403)
  • Loading branch information
@dnhatn
dnhatn committed Jun 28, 2018
2 parents e258970 + 840b768 commit 8e245a3
Show file tree
Hide file tree
Showing 217 changed files with 2,356 additions and 1,160 deletions.
9 changes: 5 additions & 4 deletions benchmarks/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,11 @@ buildscript {
}

apply plugin: 'elasticsearch.build'
// build an uberjar with all benchmarks
apply plugin: 'com.github.johnrengelman.shadow'
// have the shadow plugin provide the runShadow task
apply plugin: 'application'

// order of this section matters, see: https://github.com/johnrengelman/shadow/issues/336
apply plugin: 'application' // have the shadow plugin provide the runShadow task
mainClassName = 'org.openjdk.jmh.Main'
apply plugin: 'com.github.johnrengelman.shadow' // build an uberjar with all benchmarks

// Not published so no need to assemble
tasks.remove(assemble)
Expand Down
28 changes: 28 additions & 0 deletions build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -320,6 +320,9 @@ gradle.projectsEvaluated {
// :test:framework:test cannot run before and after :server:test
return
}
if (tasks.findByPath('test') != null && tasks.findByPath('integTest') != null) {
integTest.mustRunAfter test
}
configurations.all { Configuration configuration ->
/*
* The featureAwarePlugin configuration has a dependency on x-pack:plugin:core and x-pack:plugin:core has a dependency on the
Expand Down Expand Up @@ -569,3 +572,28 @@ gradle.projectsEvaluated {
}
}
}

if (System.properties.get("build.compare") != null) {
apply plugin: 'compare-gradle-builds'
compareGradleBuilds {
ext.referenceProject = System.properties.get("build.compare")
doFirst {
if (file(referenceProject).exists() == false) {
throw new GradleException(
"Use git worktree to check out a version to compare against to ../elasticsearch_build_reference"
)
}
}
sourceBuild {
gradleVersion = "4.7" // does not default to gradle weapper of project dir, but current version
projectDir = referenceProject
tasks = ["clean", "assemble"]
arguments = ["-Dbuild.compare_friendly=true"]
}
targetBuild {
tasks = ["clean", "assemble"]
// use -Dorg.gradle.java.home= to alter jdk versions
arguments = ["-Dbuild.compare_friendly=true"]
}
}
}
30 changes: 11 additions & 19 deletions buildSrc/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,13 @@
* under the License.
*/

import java.nio.file.Files

import org.gradle.util.GradleVersion
import java.nio.file.Files

apply plugin: 'groovy'
plugins {
id 'java-gradle-plugin'
id 'groovy'
}

group = 'org.elasticsearch.gradle'

Expand Down Expand Up @@ -83,9 +85,10 @@ repositories {
}

dependencies {
compile gradleApi()
compile localGroovy()
compile "com.carrotsearch.randomizedtesting:junit4-ant:${props.getProperty('randomizedrunner')}"
compile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${props.getProperty('randomizedrunner')}"

compile("junit:junit:${props.getProperty('junit')}") {
transitive = false
}
Expand All @@ -97,31 +100,30 @@ dependencies {
compile 'de.thetaphi:forbiddenapis:2.5'
compile 'org.apache.rat:apache-rat:0.11'
compile "org.elasticsearch:jna:4.5.1"
testCompile "junit:junit:${props.getProperty('junit')}"
}


// Gradle 2.14+ removed ProgressLogger(-Factory) classes from the public APIs
// Use logging dependency instead
// Gradle 4.3.1 stopped releasing the logging jars to jcenter, just use the last available one
GradleVersion logVersion = GradleVersion.current() > GradleVersion.version('4.3') ? GradleVersion.version('4.3') : GradleVersion.current()

dependencies {
compileOnly "org.gradle:gradle-logging:${logVersion.getVersion()}"
compile 'ru.vyarus:gradle-animalsniffer-plugin:1.2.0' // Gradle 2.14 requires a version > 1.0.1
}

/*****************************************************************************
* Bootstrap repositories *
*****************************************************************************/
// this will only happen when buildSrc is built on its own during build init
if (project == rootProject) {

repositories {
if (System.getProperty("repos.mavenLocal") != null) {
mavenLocal()
}
mavenCentral()
}
test.exclude 'org/elasticsearch/test/NamingConventionsCheckBadClasses*'
}

/*****************************************************************************
Expand All @@ -146,9 +148,6 @@ if (project != rootProject) {
jarHell.enabled = false
thirdPartyAudit.enabled = false

// test for elasticsearch.build tries to run with ES...
test.enabled = false

// TODO: re-enable once randomizedtesting gradle code is published and removed from here
licenseHeaders.enabled = false

Expand All @@ -159,14 +158,7 @@ if (project != rootProject) {
}

namingConventions {
testClass = 'org.elasticsearch.test.NamingConventionsCheckBadClasses$UnitTestCase'
integTestClass = 'org.elasticsearch.test.NamingConventionsCheckBadClasses$IntegTestCase'
}

task namingConventionsMain(type: org.elasticsearch.gradle.precommit.NamingConventionsTask) {
checkForTestsInMain = true
testClass = namingConventions.testClass
integTestClass = namingConventions.integTestClass
testClass = 'org.elasticsearch.gradle.test.GradleUnitTestCase'
integTestClass = 'org.elasticsearch.gradle.test.GradleIntegrationTestCase'
}
precommit.dependsOn namingConventionsMain
}
63 changes: 45 additions & 18 deletions buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingPlugin.groovy
View file
Original file line number Diff line number Diff line change
@@ -1,20 +1,44 @@
package com.carrotsearch.gradle.junit4

import com.carrotsearch.ant.tasks.junit4.JUnit4
import org.gradle.api.AntBuilder
import org.gradle.api.GradleException
import org.gradle.api.Plugin
import org.gradle.api.Project
import org.gradle.api.Task
import org.gradle.api.UnknownTaskException
import org.gradle.api.plugins.JavaBasePlugin
import org.gradle.api.tasks.TaskContainer
import org.gradle.api.tasks.TaskProvider
import org.gradle.api.tasks.testing.Test

import java.util.concurrent.atomic.AtomicBoolean

class RandomizedTestingPlugin implements Plugin<Project> {

static private AtomicBoolean sanityCheckConfigured = new AtomicBoolean(false)

void apply(Project project) {
setupSeed(project)
replaceTestTask(project.tasks)
configureAnt(project.ant)
configureSanityCheck(project)
}

private static void configureSanityCheck(Project project) {
// Check the task graph to confirm tasks were indeed replaced
// https://github.com/elastic/elasticsearch/issues/31324
if (sanityCheckConfigured.getAndSet(true) == false) {
project.rootProject.getGradle().getTaskGraph().whenReady {
List<Task> nonConforming = project.getGradle().getTaskGraph().allTasks
.findAll { it.name == "test" }
.findAll { (it instanceof RandomizedTestingTask) == false}
.collect { "${it.path} -> ${it.class}" }
if (nonConforming.isEmpty() == false) {
throw new GradleException("Found the ${nonConforming.size()} `test` tasks:" +
"\n ${nonConforming.join("\n ")}")
}
}
}
}

/**
Expand Down Expand Up @@ -45,29 +69,32 @@ class RandomizedTestingPlugin implements Plugin<Project> {
}

static void replaceTestTask(TaskContainer tasks) {
Test oldTestTask = tasks.findByPath('test')
if (oldTestTask == null) {
// Gradle 4.8 introduced lazy tasks, thus we deal both with the `test` task as well as it's provider
// https://github.com/gradle/gradle/issues/5730#issuecomment-398822153
// since we can't be sure if the task was ever realized, we remove both the provider and the task
TaskProvider<Test> oldTestProvider
try {
oldTestProvider = tasks.getByNameLater(Test, 'test')
} catch (UnknownTaskException unused) {
// no test task, ok, user will use testing task on their own
return
}
tasks.remove(oldTestTask)
Test oldTestTask = oldTestProvider.get()

Map properties = [
name: 'test',
type: RandomizedTestingTask,
dependsOn: oldTestTask.dependsOn,
group: JavaBasePlugin.VERIFICATION_GROUP,
description: 'Runs unit tests with the randomized testing framework'
]
RandomizedTestingTask newTestTask = tasks.create(properties)
newTestTask.classpath = oldTestTask.classpath
newTestTask.testClassesDir = oldTestTask.project.sourceSets.test.output.classesDir
// since gradle 4.5, tasks immutable dependencies are "hidden" (do not show up in dependsOn)
// so we must explicitly add a dependency on generating the test classpath
newTestTask.dependsOn('testClasses')
// we still have to use replace here despite the remove above because the task container knows about the provider
// by the same name
RandomizedTestingTask newTestTask = tasks.replace('test', RandomizedTestingTask)
newTestTask.configure{
group = JavaBasePlugin.VERIFICATION_GROUP
description = 'Runs unit tests with the randomized testing framework'
dependsOn oldTestTask.dependsOn, 'testClasses'
classpath = oldTestTask.classpath
testClassesDirs = oldTestTask.project.sourceSets.test.output.classesDirs
}

// hack so check task depends on custom test
Task checkTask = tasks.findByPath('check')
Task checkTask = tasks.getByName('check')
checkTask.dependsOn.remove(oldTestProvider)
checkTask.dependsOn.remove(oldTestTask)
checkTask.dependsOn.add(newTestTask)
}
Expand Down
22 changes: 11 additions & 11 deletions buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingTask.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,18 +6,20 @@ import groovy.xml.NamespaceBuilder
import groovy.xml.NamespaceBuilderSupport
import org.apache.tools.ant.BuildException
import org.apache.tools.ant.DefaultLogger
import org.apache.tools.ant.Project
import org.apache.tools.ant.RuntimeConfigurable
import org.apache.tools.ant.UnknownElement
import org.elasticsearch.gradle.BuildPlugin
import org.gradle.api.DefaultTask
import org.gradle.api.InvalidUserDataException
import org.gradle.api.file.FileCollection
import org.gradle.api.file.FileTreeElement
import org.gradle.api.internal.tasks.options.Option
import org.gradle.api.specs.Spec
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.InputDirectory
import org.gradle.api.tasks.Optional
import org.gradle.api.tasks.TaskAction
import org.gradle.api.tasks.options.Option
import org.gradle.api.tasks.util.PatternFilterable
import org.gradle.api.tasks.util.PatternSet
import org.gradle.internal.logging.progress.ProgressLoggerFactory
Expand All @@ -43,8 +45,8 @@ class RandomizedTestingTask extends DefaultTask {
@Input
String parallelism = '1'

@InputDirectory
File testClassesDir
@Input
FileCollection testClassesDirs

@Optional
@Input
Expand Down Expand Up @@ -220,15 +222,15 @@ class RandomizedTestingTask extends DefaultTask {
listener = new DefaultLogger(
errorPrintStream: System.err,
outputPrintStream: System.out,
messageOutputLevel: org.apache.tools.ant.Project.MSG_INFO)
messageOutputLevel: Project.MSG_INFO)
} else {
// we want to buffer the info, and emit it if the test fails
antLoggingBuffer = new ByteArrayOutputStream()
PrintStream stream = new PrintStream(antLoggingBuffer, true, "UTF-8")
listener = new DefaultLogger(
errorPrintStream: stream,
outputPrintStream: stream,
messageOutputLevel: org.apache.tools.ant.Project.MSG_INFO)
messageOutputLevel: Project.MSG_INFO)
}
project.ant.project.addBuildListener(listener)
}
Expand All @@ -251,12 +253,10 @@ class RandomizedTestingTask extends DefaultTask {
if (argLine != null) {
jvmarg(line: argLine)
}
fileset(dir: testClassesDir) {
for (String includePattern : patternSet.getIncludes()) {
include(name: includePattern)
}
for (String excludePattern : patternSet.getExcludes()) {
exclude(name: excludePattern)
testClassesDirs.each { testClassDir ->
fileset(dir: testClassDir) {
patternSet.getIncludes().each { include(name: it) }
patternSet.getExcludes().each { exclude(name: it) }
}
}
for (Map.Entry<String, Object> prop : systemProperties) {
Expand Down
28 changes: 19 additions & 9 deletions buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -348,7 +348,9 @@ class BuildPlugin implements Plugin<Project> {
// just a self contained test-fixture configuration, likely transitive and hellacious
return
}
configuration.resolutionStrategy.failOnVersionConflict()
configuration.resolutionStrategy {
failOnVersionConflict()
}
})

// force all dependencies added directly to compile/testCompile to be non-transitive, except for ES itself
Expand Down Expand Up @@ -475,13 +477,17 @@ class BuildPlugin implements Plugin<Project> {
}
}

project.tasks.withType(GenerateMavenPom.class) { GenerateMavenPom t ->
// place the pom next to the jar it is for
t.destination = new File(project.buildDir, "distributions/${project.archivesBaseName}-${project.version}.pom")
// build poms with assemble (if the assemble task exists)
Task assemble = project.tasks.findByName('assemble')
if (assemble) {
assemble.dependsOn(t)
// Work around Gradle 4.8 issue until we `enableFeaturePreview('STABLE_PUBLISHING')`
// https://github.com/gradle/gradle/issues/5696#issuecomment-396965185
project.getGradle().getTaskGraph().whenReady {
project.tasks.withType(GenerateMavenPom.class) { GenerateMavenPom t ->
// place the pom next to the jar it is for
t.destination = new File(project.buildDir, "distributions/${project.archivesBaseName}-${project.version}.pom")
// build poms with assemble (if the assemble task exists)
Task assemble = project.tasks.findByName('assemble')
if (assemble) {
assemble.dependsOn(t)
}
}
}
}
Expand Down Expand Up @@ -625,6 +631,10 @@ class BuildPlugin implements Plugin<Project> {
jarTask.manifest.attributes('Change': shortHash)
}
}
// Force manifest entries that change by nature to a constant to be able to compare builds more effectively
if (System.properties.getProperty("build.compare_friendly", "false") == "true") {
jarTask.manifest.getAttributes().clear()
}
}
// add license/notice files
project.afterEvaluate {
Expand Down Expand Up @@ -741,7 +751,7 @@ class BuildPlugin implements Plugin<Project> {
project.extensions.add('additionalTest', { String name, Closure config ->
RandomizedTestingTask additionalTest = project.tasks.create(name, RandomizedTestingTask.class)
additionalTest.classpath = test.classpath
additionalTest.testClassesDir = test.testClassesDir
additionalTest.testClassesDirs = test.testClassesDirs
additionalTest.configure(commonTestConfig(project))
additionalTest.configure(config)
test.dependsOn(additionalTest)
Expand Down
Loading

0 comments on commit 8e245a3

Please sign in to comment.