Skip to content

Commit

Permalink
Merge remote-tracking branch 'es/master' into ccr
Browse files Browse the repository at this point in the history 
* es/master: (143 commits)
  Revert "Disable BWC tests for build issues"
  Remove AcknowledgedRestListener in favour of RestToXContentListener (#28724)
  Build: Consolidate archives and packages configuration (#28760)
  Skip some plugins service tests on Windows
  Migrate some *ResponseTests to AbstractStreamableXContentTestCase (#28749)
  Disable BWC tests for build issues
  Ensure that azure stream has socket privileges (#28751)
  [DOCS] Fixed broken link.
  Pass InputStream when creating XContent parser (#28754)
  [DOCS] Changed to use transient setting to reenabled allocation. Closes #27677
  Delay path expansion on Windows
  [TEST] replace randomAsciiAlphanumOfLengthBetween with randomAsciiLettersOfLengthBetween
  [Tests] Extract the testing logic for Google Cloud Storage (#28576)
  [Docs] Update links to java9 docs (#28750)
  version set in ingest pipeline (#27573)
  Revert "Add startup logging for standalone tests"
  Tests: don't wait for completion while trying to get completed task
  Add 5.6.9 snapshot version
  [Docs] Java high-level REST client : clean up (#28703)
  Updated distribution outputs in contributing docs
  ...
  • Loading branch information
@martijnvg
martijnvg committed Feb 22, 2018
2 parents ed014e5 + daf430c commit f6b46db
Show file tree
Hide file tree
Showing 569 changed files with 9,501 additions and 5,529 deletions.
7 changes: 6 additions & 1 deletion CONTRIBUTING.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,12 @@ cd elasticsearch/
./gradlew assemble
```

You will find the newly built packages under: `./distribution/(deb|rpm|tar|zip)/build/distributions/`.
The package distributions (Debian and RPM) can be found under:
`./distribution/packages/(deb|rpm)/build/distributions/`

The archive distributions (tar and zip) can be found under:
`./distribution/archives/(tar|zip)/build/distributions/`


Before submitting your changes, run the test suite to make sure that nothing is broken, with:

Expand Down
2 changes: 1 addition & 1 deletion README.textile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -192,7 +192,7 @@ h3. Distributed, Highly Available

Let's face it, things will fail....

Elasticsearch is a highly available and distributed search engine. Each index is broken down into shards, and each shard can have one or more replica. By default, an index is created with 5 shards and 1 replica per shard (5/1). There are many topologies that can be used, including 1/10 (improve search performance), or 20/1 (improve indexing performance, with search executed in a map reduce fashion across shards).
Elasticsearch is a highly available and distributed search engine. Each index is broken down into shards, and each shard can have one or more replicas. By default, an index is created with 5 shards and 1 replica per shard (5/1). There are many topologies that can be used, including 1/10 (improve search performance), or 20/1 (improve indexing performance, with search executed in a map reduce fashion across shards).

In order to play with the distributed nature of Elasticsearch, simply bring more nodes up and shut down nodes. The system will continue to serve requests (make sure you use the correct http port) with the latest data indexed.

Expand Down
2 changes: 2 additions & 0 deletions TESTING.asciidoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -264,6 +264,8 @@ The REST layer is tested through specific tests that are shared between all
the elasticsearch official clients and consist of YAML files that describe the
operations to be executed and the obtained results that need to be tested.

The YAML files support various operators defined in the link:/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc[rest-api-spec] and adhere to the link:/rest-api-spec/README.markdown[Elasticsearch REST API JSON specification]

The REST tests are run automatically when executing the "./gradlew check" command. To run only the
REST tests use the following command:

Expand Down
50 changes: 31 additions & 19 deletions build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ allprojects {
isIdea = System.getProperty("idea.active") != null || gradle.startParameter.taskNames.contains('idea') || gradle.startParameter.taskNames.contains('cleanIdea')

// for BWC testing
versionCollection = versions
bwcVersions = versions

buildMetadata = buildMetadataMap
}
Expand All @@ -122,13 +122,13 @@ task verifyVersions {
Set<Version> knownVersions = new TreeSet<>(xml.versioning.versions.version.collect { it.text() }.findAll { it ==~ /\d\.\d\.\d/ }.collect { Version.fromString(it) })

// Limit the known versions to those that should be index compatible, and are not future versions
knownVersions = knownVersions.findAll { it.major >= versions.currentVersion.major - 1 && it.before(VersionProperties.elasticsearch) }
knownVersions = knownVersions.findAll { it.major >= bwcVersions.currentVersion.major - 1 && it.before(VersionProperties.elasticsearch) }

/* Limit the listed versions to those that have been marked as released.
* Versions not marked as released don't get the same testing and we want
* to make sure that we flip all unreleased versions to released as soon
* as possible after release. */
Set<Version> actualVersions = new TreeSet<>(versions.versionsIndexCompatibleWithCurrent.findAll { false == it.snapshot })
Set<Version> actualVersions = new TreeSet<>(bwcVersions.indexCompatible.findAll { false == it.snapshot })

// Finally, compare!
if (knownVersions.equals(actualVersions) == false) {
Expand All @@ -144,13 +144,24 @@ task verifyVersions {
* the enabled state of every bwc task. It should be set back to true
* after the backport of the backcompat code is complete.
*/
allprojects {
ext.bwc_tests_enabled = true
final boolean bwc_tests_enabled = true
final String bwc_tests_disabled_issue = "" /* place a PR link here when commiting bwc changes */
if (bwc_tests_enabled == false) {
if (bwc_tests_disabled_issue.isEmpty()) {
throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")
}
println "========================= WARNING ========================="
println " Backwards compatibility tests are disabled!"
println "See ${bwc_tests_disabled_issue}"
println "==========================================================="
}
subprojects {
ext.bwc_tests_enabled = bwc_tests_enabled
}

task verifyBwcTestsEnabled {
doLast {
if (project.bwc_tests_enabled == false) {
if (bwc_tests_enabled == false) {
throw new GradleException('Bwc tests are disabled. They must be re-enabled after completing backcompat behavior backporting.')
}
}
Expand Down Expand Up @@ -192,11 +203,11 @@ subprojects {
"org.elasticsearch.client:test:${version}": ':client:test',
"org.elasticsearch.client:transport:${version}": ':client:transport',
"org.elasticsearch.test:framework:${version}": ':test:framework',
"org.elasticsearch.distribution.integ-test-zip:elasticsearch:${version}": ':distribution:integ-test-zip',
"org.elasticsearch.distribution.zip:elasticsearch:${version}": ':distribution:zip',
"org.elasticsearch.distribution.tar:elasticsearch:${version}": ':distribution:tar',
"org.elasticsearch.distribution.rpm:elasticsearch:${version}": ':distribution:rpm',
"org.elasticsearch.distribution.deb:elasticsearch:${version}": ':distribution:deb',
"org.elasticsearch.distribution.integ-test-zip:elasticsearch:${version}": ':distribution:archives:integ-test-zip',
"org.elasticsearch.distribution.zip:elasticsearch:${version}": ':distribution:archives:zip',
"org.elasticsearch.distribution.tar:elasticsearch:${version}": ':distribution:archives:tar',
"org.elasticsearch.distribution.rpm:elasticsearch:${version}": ':distribution:packages:rpm',
"org.elasticsearch.distribution.deb:elasticsearch:${version}": ':distribution:packages:deb',
"org.elasticsearch.test:logger-usage:${version}": ':test:logger-usage',
// for transport client
"org.elasticsearch.plugin:transport-netty4-client:${version}": ':modules:transport-netty4',
Expand All @@ -208,13 +219,14 @@ subprojects {
"org.elasticsearch.plugin:rank-eval-client:${version}": ':modules:rank-eval',
]

for (final Version version : versionCollection.versionsIndexCompatibleWithCurrent) {
if (version.branch != null) {
final String snapshotProject = ":distribution:bwc-snapshot-${version.branch}"
project(snapshotProject).ext.bwcVersion = version
ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch:${version}"] = snapshotProject
ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch:${version}"] = snapshotProject
ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${version}"] = snapshotProject
bwcVersions.snapshotProjectNames.each { snapshotName ->
Version snapshot = bwcVersions.getSnapshotForProject(snapshotName)
if (snapshot != null ) {
String snapshotProject = ":distribution:bwc:${snapshotName}"
project(snapshotProject).ext.bwcVersion = snapshot
ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch:${snapshot}"] = snapshotProject
ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch:${snapshot}"] = snapshotProject
ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${snapshot}"] = snapshotProject
}
}

Expand Down Expand Up @@ -245,7 +257,7 @@ subprojects {
}
}
project.configurations.compile.dependencies.findAll().toSorted(sortClosure).each(depJavadocClosure)
project.configurations.provided.dependencies.findAll().toSorted(sortClosure).each(depJavadocClosure)
project.configurations.compileOnly.dependencies.findAll().toSorted(sortClosure).each(depJavadocClosure)
}
}
}
Expand Down
155 changes: 70 additions & 85 deletions buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/TestProgressLogger.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,14 @@ package com.carrotsearch.gradle.junit4

import com.carrotsearch.ant.tasks.junit4.JUnit4
import com.carrotsearch.ant.tasks.junit4.dependencies.com.google.common.eventbus.Subscribe
import com.carrotsearch.ant.tasks.junit4.events.TestStartedEvent
import com.carrotsearch.ant.tasks.junit4.events.aggregated.AggregatedQuitEvent
import com.carrotsearch.ant.tasks.junit4.events.aggregated.AggregatedStartEvent
import com.carrotsearch.ant.tasks.junit4.events.aggregated.AggregatedSuiteResultEvent
import com.carrotsearch.ant.tasks.junit4.events.aggregated.AggregatedSuiteStartedEvent
import com.carrotsearch.ant.tasks.junit4.events.aggregated.AggregatedTestResultEvent
import com.carrotsearch.ant.tasks.junit4.events.aggregated.ChildBootstrap
import com.carrotsearch.ant.tasks.junit4.events.aggregated.HeartBeatEvent
import com.carrotsearch.ant.tasks.junit4.listeners.AggregatedEventListener
import org.gradle.internal.logging.progress.ProgressLogger
import org.gradle.internal.logging.progress.ProgressLoggerFactory
Expand All @@ -34,7 +39,6 @@ import static com.carrotsearch.ant.tasks.junit4.events.aggregated.TestStatus.FAI
import static com.carrotsearch.ant.tasks.junit4.events.aggregated.TestStatus.IGNORED
import static com.carrotsearch.ant.tasks.junit4.events.aggregated.TestStatus.IGNORED_ASSUMPTION
import static com.carrotsearch.ant.tasks.junit4.events.aggregated.TestStatus.OK
import static java.lang.Math.max

/**
* Adapts junit4's event listeners into gradle's ProgressLogger. Note that
Expand All @@ -54,137 +58,118 @@ import static java.lang.Math.max
class TestProgressLogger implements AggregatedEventListener {
/** Factory to build a progress logger when testing starts */
ProgressLoggerFactory factory
ProgressLogger progressLogger
ProgressLogger parentProgressLogger
ProgressLogger suiteLogger
ProgressLogger testLogger
ProgressLogger[] slaveLoggers
int totalSuites
int totalSlaves

// sprintf formats used to align the integers we print
String suitesFormat
String slavesFormat
String testsFormat

// Counters incremented test completion.
volatile int suitesCompleted = 0
volatile int testsCompleted = 0
volatile int testsFailed = 0
volatile int testsIgnored = 0

// Information about the last, most interesting event.
volatile String eventDescription
volatile int eventSlave
volatile long eventExecutionTime

/** Have we finished a whole suite yet? */
volatile boolean suiteFinished = false
/* Note that we probably overuse volatile here but it isn't hurting us and
lets us move things around without worrying about breaking things. */

@Subscribe
void onStart(AggregatedStartEvent e) throws IOException {
totalSuites = e.suiteCount
totalSlaves = e.slaveCount
progressLogger = factory.newOperation(TestProgressLogger)
progressLogger.setDescription('Randomized test runner')
progressLogger.started()
progressLogger.progress(
"Starting JUnit4 for ${totalSuites} suites on ${totalSlaves} jvms")

suitesFormat = "%0${widthForTotal(totalSuites)}d"
slavesFormat = "%-${widthForTotal(totalSlaves)}s"
/* Just guess the number of tests because we can't figure it out from
here and it isn't worth doing anything fancy to prevent the console
from jumping around a little. 200 is a pretty wild guess for the
minimum but it makes REST tests output sanely. */
int totalNumberOfTestsGuess = max(200, totalSuites * 10)
testsFormat = "%0${widthForTotal(totalNumberOfTestsGuess)}d"
parentProgressLogger = factory.newOperation(TestProgressLogger)
parentProgressLogger.setDescription('Randomized test runner')
parentProgressLogger.started()

suiteLogger = factory.newOperation(TestProgressLogger, parentProgressLogger)
suiteLogger.setDescription('Suite logger')
suiteLogger.started("Suites: 0/" + totalSuites)
testLogger = factory.newOperation(TestProgressLogger, parentProgressLogger)
testLogger.setDescription('Test logger')
testLogger.started('Tests: completed: 0, failed: 0, ignored: 0')
slaveLoggers = new ProgressLogger[e.slaveCount]
for (int i = 0; i < e.slaveCount; ++i) {
slaveLoggers[i] = factory.newOperation(TestProgressLogger, parentProgressLogger)
slaveLoggers[i].setDescription("J${i} test logger")
slaveLoggers[i].started("J${i}: initializing...")
}
}

@Subscribe
void onChildBootstrap(ChildBootstrap e) throws IOException {
slaveLoggers[e.getSlave().id].progress("J${e.slave.id}: starting (pid ${e.slave.pidString})")
}

@Subscribe
void onQuit(AggregatedQuitEvent e) throws IOException {
suiteLogger.completed()
testLogger.completed()
for (ProgressLogger slaveLogger : slaveLoggers) {
slaveLogger.completed()
}
parentProgressLogger.completed()
}

@Subscribe
void onSuiteStart(AggregatedSuiteStartedEvent e) throws IOException {
String suiteName = simpleName(e.suiteStartedEvent.description.className)
slaveLoggers[e.slave.id].progress("J${e.slave.id}: ${suiteName} - initializing")
}

@Subscribe
void onSuiteResult(AggregatedSuiteResultEvent e) throws IOException {
suitesCompleted++
suiteLogger.progress("Suites: " + suitesCompleted + "/" + totalSuites)
}

@Subscribe
void onTestResult(AggregatedTestResultEvent e) throws IOException {
final String statusMessage
testsCompleted++
switch (e.status) {
case ERROR:
case FAILURE:
testsFailed++
statusMessage = "failed"
break
case IGNORED:
case IGNORED_ASSUMPTION:
testsIgnored++
statusMessage = "ignored"
break
case OK:
String time = formatDurationInSeconds(e.executionTime)
statusMessage = "completed [${time}]"
break
default:
throw new IllegalArgumentException(
"Unknown test status: [${e.status}]")
throw new IllegalArgumentException("Unknown test status: [${e.status}]")
}
if (!suiteFinished) {
updateEventInfo(e)
}

log()
testLogger.progress("Tests: completed: ${testsCompleted}, failed: ${testsFailed}, ignored: ${testsIgnored}")
String testName = simpleName(e.description.className) + '.' + e.description.methodName
slaveLoggers[e.slave.id].progress("J${e.slave.id}: ${testName} ${statusMessage}")
}

@Subscribe
void onSuiteResult(AggregatedSuiteResultEvent e) throws IOException {
suitesCompleted++
suiteFinished = true
updateEventInfo(e)
log()
void onTestStarted(TestStartedEvent e) throws IOException {
String testName = simpleName(e.description.className) + '.' + e.description.methodName
slaveLoggers[e.slave.id].progress("J${e.slave.id}: ${testName} ...")
}

/**
* Update the suite information with a junit4 event.
*/
private void updateEventInfo(Object e) {
eventDescription = simpleName(e.description.className)
if (e.description.methodName != null) {
eventDescription += "#${e.description.methodName}"
}
eventSlave = e.slave.id
eventExecutionTime = e.executionTime
@Subscribe
void onHeartbeat(HeartBeatEvent e) throws IOException {
String testName = simpleName(e.description.className) + '.' + e.description.methodName
String time = formatDurationInSeconds(e.getNoEventDuration())
slaveLoggers[e.slave.id].progress("J${e.slave.id}: ${testName} stalled for ${time}")
}

/**
* Extract a Class#getSimpleName style name from Class#getName style
* string. We can't just use Class#getSimpleName because junit descriptions
* don't alway s set the class field but they always set the className
* don't always set the class field but they always set the className
* field.
*/
private static String simpleName(String className) {
return className.substring(className.lastIndexOf('.') + 1)
}

private void log() {
/* Remember that instances of this class are only ever active on one
thread at a time so there really aren't race conditions here. It'd be
OK if there were because they'd only display an overcount
temporarily. */
String log = ''
if (totalSuites > 1) {
/* Skip printing the suites to save space when there is only a
single suite. This is nice because when there is only a single
suite we log the method name and those can be long. */
log += sprintf("Suites [${suitesFormat}/${suitesFormat}], ",
[suitesCompleted, totalSuites])
}
log += sprintf("Tests [${testsFormat}|%d|%d], ",
[testsCompleted, testsFailed, testsIgnored])
log += "in ${formatDurationInSeconds(eventExecutionTime)} "
if (totalSlaves > 1) {
/* Skip printing the slaves if there is only one of them. This is
nice because when there is only a single slave there is often
only a single suite and we could use the extra space to log the
test method names. */
log += "J${sprintf(slavesFormat, eventSlave)} "
}
log += "completed ${eventDescription}"
progressLogger.progress(log)
}

private static int widthForTotal(int total) {
return ((total - 1) as String).length()
}

@Override
void setOuter(JUnit4 junit) {}
}
8 changes: 5 additions & 3 deletions buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,6 @@ class BuildPlugin implements Plugin<Project> {
project.pluginManager.apply('nebula.info-java')
project.pluginManager.apply('nebula.info-scm')
project.pluginManager.apply('nebula.info-jar')
project.pluginManager.apply(ProvidedBasePlugin)

globalBuildInfo(project)
configureRepositories(project)
Expand Down Expand Up @@ -261,6 +260,9 @@ class BuildPlugin implements Plugin<Project> {
* to iterate the transitive dependencies and add excludes.
*/
static void configureConfigurations(Project project) {
// we want to test compileOnly deps!
project.configurations.testCompile.extendsFrom(project.configurations.compileOnly)

// we are not shipping these jars, we act like dumb consumers of these things
if (project.path.startsWith(':test:fixtures') || project.path == ':build-tools') {
return
Expand Down Expand Up @@ -297,7 +299,7 @@ class BuildPlugin implements Plugin<Project> {

project.configurations.compile.dependencies.all(disableTransitiveDeps)
project.configurations.testCompile.dependencies.all(disableTransitiveDeps)
project.configurations.provided.dependencies.all(disableTransitiveDeps)
project.configurations.compileOnly.dependencies.all(disableTransitiveDeps)
}

/** Adds repositories used by ES dependencies */
Expand Down Expand Up @@ -665,7 +667,7 @@ class BuildPlugin implements Plugin<Project> {
// only require dependency licenses for non-elasticsearch deps
project.dependencyLicenses.dependencies = project.configurations.runtime.fileCollection {
it.group.startsWith('org.elasticsearch') == false
} - project.configurations.provided
} - project.configurations.compileOnly
}

private static configureDependenciesInfo(Project project) {
Expand Down

0 comments on commit f6b46db

Please sign in to comment.