Skip to content

Commit

Permalink
Merge branch 'master' into pr/24856
Browse files Browse the repository at this point in the history 
* master: (38 commits)
  Fix Lucene version expectation
  Verify Lucene version constants
  Avoid double decrement on current query counter
  Remove the need for _UNRELEASED suffix in versions (elastic#24798)
  Adjust available and free bytes to be non-negative on huge FSes
  Begin replacing static index tests with full restart tests (elastic#24846)
  Fix plugin docs for using custom config dir
  Update context-suggest.asciidoc
  Move BWC version to 5.5 after backport
  Support Multiple Collapse Inner Hits
  Scripting: Rename CompiledType to FactoryType in ScriptContext (elastic#24897)
  Scripting: Make contexts available to ScriptEngine construction (elastic#24896)
  Mute index and relocate concurrently
  Build: Add back explicit exclusions and remove gradle exclusions (elastic#24879)
  Scripting: Move context definitions to instance type classes (elastic#24883)
  Build: Fix hadoop integ test error on windows (elastic#24885)
  Put mapping and index template requests do not need content type detection for 5.3.0+ (elastic#24835)
  Add the ability to store objects with a ScrollContext (elastic#24777)
  add docs example for Ingest scripts manipulating document metadata (elastic#24875)
  Fix error message if an incompatible node connects (elastic#24884)
  ...
  • Loading branch information
@jasontedor
jasontedor committed May 28, 2017
2 parents 2b3f34c + d1318e4 commit 05b11b0
Show file tree
Hide file tree
Showing 200 changed files with 5,164 additions and 1,637 deletions.
67 changes: 58 additions & 9 deletions build.gradle
View file
Expand Up @@ -62,7 +62,12 @@ configure(subprojects.findAll { it.projectDir.toPath().startsWith(rootPath) }) {
}
}

// introspect all versions of ES that may be tested agains for backwards compatibility
/* Introspect all versions of ES that may be tested agains for backwards
* compatibility. It is *super* important that this logic is the same as the
* logic in VersionUtils.java, modulo alphas, betas, and rcs which are ignored
* in gradle because they don't have any backwards compatibility guarantees
* but are not ignored in VersionUtils.java because the tests expect them not
* to be. */
Version currentVersion = Version.fromString(VersionProperties.elasticsearch.minus('-SNAPSHOT'))
int prevMajor = currentVersion.major - 1
File versionFile = file('core/src/main/java/org/elasticsearch/Version.java')
Expand All @@ -72,13 +77,14 @@ List<Version> versions = []
int prevMinorIndex = -1 // index in the versions list of the last minor from the prev major
int lastPrevMinor = -1 // the minor version number from the prev major we most recently seen
for (String line : versionLines) {
Matcher match = line =~ /\W+public static final Version V_(\d+)_(\d+)_(\d+)(_UNRELEASED)? .*/
/* Note that this skips alphas and betas which is fine because they aren't
* compatible with anything. */
Matcher match = line =~ /\W+public static final Version V_(\d+)_(\d+)_(\d+) .*/
if (match.matches()) {
int major = Integer.parseInt(match.group(1))
int minor = Integer.parseInt(match.group(2))
int bugfix = Integer.parseInt(match.group(3))
boolean unreleased = match.group(4) != null
Version foundVersion = new Version(major, minor, bugfix, false, unreleased)
Version foundVersion = new Version(major, minor, bugfix, false)
if (currentVersion != foundVersion) {
versions.add(foundVersion)
}
Expand All @@ -96,10 +102,13 @@ if (currentVersion.bugfix == 0) {
// If on a release branch, after the initial release of that branch, the bugfix version will
// be bumped, and will be != 0. On master and N.x branches, we want to test against the
// unreleased version of closest branch. So for those cases, the version includes -SNAPSHOT,
// and the bwc-zip distribution will checkout and build that version.
// and the bwc distribution will checkout and build that version.
Version last = versions[-1]
versions[-1] = new Version(last.major, last.minor, last.bugfix,
true, last.unreleased)
versions[-1] = new Version(last.major, last.minor, last.bugfix, true)
if (last.bugfix == 0) {
versions[-2] = new Version(
versions[-2].major, versions[-2].minor, versions[-2].bugfix, true)
}
}

// injecting groovy property variables into all projects
Expand All @@ -114,6 +123,44 @@ allprojects {
}
}

task('verifyVersions') {
description 'Verifies that all released versions that are indexed compatible are listed in Version.java.'
group 'Verification'
enabled = false == gradle.startParameter.isOffline()
doLast {
// Read the list from maven central
Node xml
new URL('https://repo1.maven.org/maven2/org/elasticsearch/elasticsearch/maven-metadata.xml').openStream().withStream { s ->
xml = new XmlParser().parse(s)
}
Set<String> knownVersions = new TreeSet<>(xml.versioning.versions.version.collect { it.text() }.findAll { it ==~ /\d\.\d\.\d/ })

// Limit the known versions to those that should be index compatible
knownVersions = knownVersions.findAll { Integer.parseInt(it.split('\\.')[0]) >= prevMajor }

/* Limit the listed versions to those that have been marked as released.
* Versions not marked as released don't get the same testing and we want
* to make sure that we flip all unreleased versions to released as soon
* as possible after release. */
Set<String> actualVersions = new TreeSet<>(
indexCompatVersions
.findAll { false == it.snapshot }
.collect { it.toString() })

// TODO this is almost certainly going to fail on 5.4 when we release 5.5.0

// Finally, compare!
if (!knownVersions.equals(actualVersions)) {
throw new GradleException("out-of-date versions\nActual :" +
actualVersions + "\nExpected:" + knownVersions +
"; update Version.java")
}
}
}
task('precommit') {
dependsOn(verifyVersions)
}

subprojects {
project.afterEvaluate {
// include license and notice in jars
Expand Down Expand Up @@ -173,8 +220,10 @@ subprojects {
]
if (wireCompatVersions[-1].snapshot) {
// if the most previous version is a snapshot, we need to connect that version to the
// bwc-zip project which will checkout and build that snapshot version
ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${wireCompatVersions[-1]}"] = ':distribution:bwc-zip'
// bwc project which will checkout and build that snapshot version
ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch:${wireCompatVersions[-1]}"] = ':distribution:bwc'
ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch:${wireCompatVersions[-1]}"] = ':distribution:bwc'
ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${wireCompatVersions[-1]}"] = ':distribution:bwc'
}
project.afterEvaluate {
configurations.all {
Expand Down
50 changes: 47 additions & 3 deletions buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
View file
Expand Up @@ -311,7 +311,13 @@ class BuildPlugin implements Plugin<Project> {
/**
* Returns a closure which can be used with a MavenPom for fixing problems with gradle generated poms.
*
* The current fixup is to set compile time deps back to compile from runtime (known issue with maven-publish plugin).
* <ul>
* <li>Remove transitive dependencies. We currently exclude all artifacts explicitly instead of using wildcards
* as Ivy incorrectly translates POMs with * excludes to Ivy XML with * excludes which results in the main artifact
* being excluded as well (see https://issues.apache.org/jira/browse/IVY-1531). Note that Gradle 2.14+ automatically
* translates non-transitive dependencies to * excludes. We should revisit this when upgrading Gradle.</li>
* <li>Set compile time deps back to compile from runtime (known issue with maven-publish plugin)</li>
* </ul>
*/
private static Closure fixupDependencies(Project project) {
return { XmlProvider xml ->
Expand All @@ -321,15 +327,53 @@ class BuildPlugin implements Plugin<Project> {
return
}

// fix deps incorrectly marked as runtime back to compile time deps
// see https://discuss.gradle.org/t/maven-publish-plugin-generated-pom-making-dependency-scope-runtime/7494/4
// check each dependency for any transitive deps
for (Node depNode : depsNodes.get(0).children()) {
String groupId = depNode.get('groupId').get(0).text()
String artifactId = depNode.get('artifactId').get(0).text()
String version = depNode.get('version').get(0).text()

// fix deps incorrectly marked as runtime back to compile time deps
// see https://discuss.gradle.org/t/maven-publish-plugin-generated-pom-making-dependency-scope-runtime/7494/4
boolean isCompileDep = project.configurations.compile.allDependencies.find { dep ->
dep.name == depNode.artifactId.text()
}
if (depNode.scope.text() == 'runtime' && isCompileDep) {
depNode.scope*.value = 'compile'
}

// remove any exclusions added by gradle, they contain wildcards and systems like ivy have bugs with wildcards
// see https://github.com/elastic/elasticsearch/issues/24490
NodeList exclusionsNode = depNode.get('exclusions')
if (exclusionsNode.size() > 0) {
depNode.remove(exclusionsNode.get(0))
}

// collect the transitive deps now that we know what this dependency is
String depConfig = transitiveDepConfigName(groupId, artifactId, version)
Configuration configuration = project.configurations.findByName(depConfig)
if (configuration == null) {
continue // we did not make this dep non-transitive
}
Set<ResolvedArtifact> artifacts = configuration.resolvedConfiguration.resolvedArtifacts
if (artifacts.size() <= 1) {
// this dep has no transitive deps (or the only artifact is itself)
continue
}

// we now know we have something to exclude, so add exclusions for all artifacts except the main one
Node exclusions = depNode.appendNode('exclusions')
for (ResolvedArtifact artifact : artifacts) {
ModuleVersionIdentifier moduleVersionIdentifier = artifact.moduleVersion.id;
String depGroupId = moduleVersionIdentifier.group
String depArtifactId = moduleVersionIdentifier.name
// add exclusions for all artifacts except the main one
if (depGroupId != groupId || depArtifactId != artifactId) {
Node exclusion = exclusions.appendNode('exclusion')
exclusion.appendNode('groupId', depGroupId)
exclusion.appendNode('artifactId', depArtifactId)
}
}
}
}
}
Expand Down
10 changes: 2 additions & 8 deletions buildSrc/src/main/groovy/org/elasticsearch/gradle/Version.groovy
View file
Expand Up @@ -29,20 +29,14 @@ public class Version {
final int bugfix
final int id
final boolean snapshot
/**
* Is the vesion listed as {@code _UNRELEASED} in Version.java.
*/
final boolean unreleased

public Version(int major, int minor, int bugfix, boolean snapshot,
boolean unreleased) {
public Version(int major, int minor, int bugfix, boolean snapshot) {
this.major = major
this.minor = minor
this.bugfix = bugfix
this.snapshot = snapshot
this.id = major * 100000 + minor * 1000 + bugfix * 10 +
(snapshot ? 1 : 0)
this.unreleased = unreleased
}

public static Version fromString(String s) {
Expand All @@ -54,7 +48,7 @@ public class Version {
bugfix = bugfix.split('-')[0]
}
return new Version(parts[0] as int, parts[1] as int, bugfix as int,
snapshot, false)
snapshot)
}

@Override
Expand Down
6 changes: 3 additions & 3 deletions buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy
View file
Expand Up @@ -46,11 +46,11 @@ class ClusterConfiguration {
int transportPort = 0

/**
* An override of the data directory. This may only be used with a single node.
* The value is lazily evaluated at runtime as a String path.
* An override of the data directory. Input is the node number and output
* is the override data directory.
*/
@Input
Object dataDir = null
Closure<String> dataDir = null

/** Optional override of the cluster name. */
@Input
Expand Down
5 changes: 1 addition & 4 deletions buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy
View file
Expand Up @@ -111,10 +111,7 @@ class NodeInfo {
homeDir = homeDir(baseDir, config.distribution, nodeVersion)
confDir = confDir(baseDir, config.distribution, nodeVersion)
if (config.dataDir != null) {
if (config.numNodes != 1) {
throw new IllegalArgumentException("Cannot set data dir for integ test with more than one node")
}
dataDir = config.dataDir
dataDir = "${config.dataDir(nodeNum)}"
} else {
dataDir = new File(homeDir, "data")
}
Expand Down
40 changes: 0 additions & 40 deletions core/build.gradle
View file
Expand Up @@ -277,43 +277,3 @@ if (isEclipse == false || project.path == ":core-tests") {
check.dependsOn integTest
integTest.mustRunAfter test
}

task('verifyVersions') {
description 'Verifies that all released versions that are indexed compatible are listed in Version.java.'
group 'Verification'
enabled = false == gradle.startParameter.isOffline()
doLast {
// Read the list from maven central
Node xml
new URL('https://repo1.maven.org/maven2/org/elasticsearch/elasticsearch/maven-metadata.xml').openStream().withStream { s ->
xml = new XmlParser().parse(s)
}
Set<String> knownVersions = new TreeSet<>(xml.versioning.versions.version.collect { it.text() }.findAll { it ==~ /\d\.\d\.\d/ })

// Limit the known versions to those that should be wire compatible
String currentVersion = versions.elasticsearch.minus('-SNAPSHOT')
int prevMajor = Integer.parseInt(currentVersion.split('\\.')[0]) - 1
if (prevMajor == 4) {
// 4 didn't exist, it was 2.
prevMajor = 2;
}
knownVersions = knownVersions.findAll { Integer.parseInt(it.split('\\.')[0]) >= prevMajor }

/* Limit the listed versions to those that have been marked as released.
* Versions not marked as released don't get the same testing and we want
* to make sure that we flip all unreleased versions to released as soon
* as possible after release. */
Set<String> actualVersions = new TreeSet<>(
indexCompatVersions
.findAll { false == it.unreleased }
.collect { it.toString() })

// Finally, compare!
if (!knownVersions.equals(actualVersions)) {
throw new GradleException("out-of-date versions\nActual :" +
actualVersions + "\nExpected:" + knownVersions +
"; update Version.java")
}
}
}
check.dependsOn(verifyVersions)

0 comments on commit 05b11b0

Please sign in to comment.