Skip to content

Commit

Permalink
Use project dependency instead of substitutions for distributions (#3…
Browse files Browse the repository at this point in the history 
…7730)

Currently integration tests which use either bwc snapshot versions or
the current version of elasticsearch depend on project substitutions to
link to the build of those artifacts. Likewise, vagrant tests use
dependency substitutions to get to bwc snapshots of rpm and debs.
This commit changes those to depend on the relevant project/configuration
and removes the dependency substitutions for distributions we do not
publish.
  • Loading branch information
@rjernst
rjernst committed Jan 24, 2019
1 parent 04c6414 commit d9d13f3
Show file tree
Hide file tree
Showing 5 changed files with 89 additions and 52 deletions.
21 changes: 0 additions & 21 deletions build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -221,14 +221,6 @@ allprojects {
"org.elasticsearch.plugin:elasticsearch-scripting-painless-spi:${version}": ':modules:lang-painless:spi',
"org.elasticsearch.test:framework:${version}": ':test:framework',
"org.elasticsearch.distribution.integ-test-zip:elasticsearch:${version}": ':distribution:archives:integ-test-zip',
"downloads.zip:elasticsearch:${version}": ':distribution:archives:zip',
"downloads.zip:elasticsearch-oss:${version}": ':distribution:archives:oss-zip',
"downloads.tar:elasticsearch:${version}": ':distribution:archives:tar',
"downloads.tar:elasticsearch-oss:${version}": ':distribution:archives:oss-tar',
"downloads.rpm:elasticsearch:${version}": ':distribution:packages:rpm',
"downloads.rpm:elasticsearch-oss:${version}": ':distribution:packages:oss-rpm',
"downloads.deb:elasticsearch:${version}": ':distribution:packages:deb',
"downloads.deb:elasticsearch-oss:${version}": ':distribution:packages:oss-deb',
"org.elasticsearch.test:logger-usage:${version}": ':test:logger-usage',
"org.elasticsearch.xpack.test:feature-aware:${version}": ':x-pack:test:feature-aware',
// for transport client
Expand All @@ -240,19 +232,6 @@ allprojects {
"org.elasticsearch.plugin:percolator-client:${version}": ':modules:percolator',
"org.elasticsearch.plugin:rank-eval-client:${version}": ':modules:rank-eval',
]
// substitute unreleased versions with projects that check out and build locally
bwcVersions.forPreviousUnreleased { VersionCollection.UnreleasedVersionInfo unreleasedVersion ->
Version unreleased = unreleasedVersion.version
String snapshotProject = ":distribution:bwc:${unreleasedVersion.gradleProjectName}"
ext.projectSubstitutions["downloads.deb:elasticsearch:${unreleased}"] = snapshotProject
ext.projectSubstitutions["downloads.rpm:elasticsearch:${unreleased}"] = snapshotProject
ext.projectSubstitutions["downloads.zip:elasticsearch:${unreleased}"] = snapshotProject
if (unreleased.onOrAfter('6.3.0')) {
ext.projectSubstitutions["downloads.deb:elasticsearch-oss:${unreleased}"] = snapshotProject
ext.projectSubstitutions["downloads.rpm:elasticsearch-oss:${unreleased}"] = snapshotProject
ext.projectSubstitutions["downloads.zip:elasticsearch-oss:${unreleased}"] = snapshotProject
}
}

/*
* Gradle only resolve project substitutions during dependency resolution but
Expand Down
34 changes: 25 additions & 9 deletions buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import org.apache.tools.ant.taskdefs.condition.Os
import org.elasticsearch.gradle.BuildPlugin
import org.elasticsearch.gradle.LoggedExec
import org.elasticsearch.gradle.Version
import org.elasticsearch.gradle.VersionCollection
import org.elasticsearch.gradle.VersionProperties
import org.elasticsearch.gradle.plugin.PluginBuildPlugin
import org.elasticsearch.gradle.plugin.PluginPropertiesExtension
Expand Down Expand Up @@ -171,6 +172,12 @@ class ClusterFormationTasks {

/** Adds a dependency on the given distribution */
static void configureDistributionDependency(Project project, String distro, Configuration configuration, String elasticsearchVersion) {
if (distro.equals("integ-test-zip")) {
// short circuit integ test so it doesn't complicate the rest of the distribution setup below
project.dependencies.add(configuration.name,
"org.elasticsearch.distribution.integ-test-zip:elasticsearch:${elasticsearchVersion}@zip")
return
}
// TEMP HACK
// The oss docs CI build overrides the distro on the command line. This hack handles backcompat until CI is updated.
if (distro.equals('oss-zip')) {
Expand All @@ -180,22 +187,31 @@ class ClusterFormationTasks {
distro = 'default'
}
// END TEMP HACK
if (['integ-test-zip', 'oss', 'default'].contains(distro) == false) {
if (['oss', 'default'].contains(distro) == false) {
throw new GradleException("Unknown distribution: ${distro} in project ${project.path}")
}
Version version = Version.fromString(elasticsearchVersion)
if (version.before('6.3.0') && distro.startsWith('oss-')) {
distro = distro.substring('oss-'.length())
}
String group = "downloads.zip"
if (distro.equals("integ-test-zip")) {
group = "org.elasticsearch.distribution.integ-test-zip"
}
String group = "downloads.zip" // dummy group, does not matter except for integ-test-zip, it is ignored by the fake ivy repo
String artifactName = 'elasticsearch'
if (distro.equals('oss') && Version.fromString(elasticsearchVersion).onOrAfter('6.3.0')) {
artifactName += '-oss'
}
project.dependencies.add(configuration.name, "${group}:${artifactName}:${elasticsearchVersion}@zip")
String snapshotProject = distro == 'oss' ? 'oss-zip' : 'zip'
Object dependency
boolean internalBuild = project.hasProperty('bwcVersions')
VersionCollection.UnreleasedVersionInfo unreleasedInfo = null
if (project.hasProperty('bwcVersions')) {
// NOTE: leniency is needed for external plugin authors using build-tools. maybe build the version compat info into build-tools?
unreleasedInfo = project.bwcVersions.unreleasedInfo(version)
}
if (unreleasedInfo != null) {
dependency = project.dependencies.project(path: ":distribution:bwc:${unreleasedInfo.gradleProjectName}", configuration: snapshotProject)
} else if (internalBuild && elasticsearchVersion.equals(VersionProperties.elasticsearch)) {
dependency = project.dependencies.project(path: ":distribution:archives:${snapshotProject}")
} else {
dependency = "${group}:${artifactName}:${elasticsearchVersion}@zip"
}
project.dependencies.add(configuration.name, dependency)
}

/** Adds a dependency on a different version of the given plugin, which will be retrieved using gradle's dependency resolution */
Expand Down
35 changes: 26 additions & 9 deletions buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import org.apache.tools.ant.taskdefs.condition.Os
import org.elasticsearch.gradle.FileContentsTask
import org.elasticsearch.gradle.LoggedExec
import org.elasticsearch.gradle.Version
import org.elasticsearch.gradle.VersionCollection
import org.gradle.api.*
import org.gradle.api.artifacts.dsl.RepositoryHandler
import org.gradle.api.execution.TaskExecutionAdapter
Expand Down Expand Up @@ -184,22 +185,38 @@ class VagrantTestPlugin implements Plugin<Project> {
upgradeFromVersion = Version.fromString(upgradeFromVersionRaw)
}

List<Object> dependencies = new ArrayList<>()
DISTRIBUTIONS.each {
// Adds a dependency for the current version
project.dependencies.add(PACKAGING_CONFIGURATION,
project.dependencies.project(path: ":distribution:${it}", configuration: 'default'))
dependencies.add(project.dependencies.project(path: ":distribution:${it}", configuration: 'default'))
}

UPGRADE_FROM_ARCHIVES.each {
// The version of elasticsearch that we upgrade *from*
project.dependencies.add(PACKAGING_CONFIGURATION,
"downloads.${it}:elasticsearch:${upgradeFromVersion}@${it}")
if (upgradeFromVersion.onOrAfter('6.3.0')) {
project.dependencies.add(PACKAGING_CONFIGURATION,
"downloads.${it}:elasticsearch-oss:${upgradeFromVersion}@${it}")
// The version of elasticsearch that we upgrade *from*
VersionCollection.UnreleasedVersionInfo unreleasedInfo = project.bwcVersions.unreleasedInfo(upgradeFromVersion)
if (unreleasedInfo != null) {
// handle snapshots pointing to bwc build
UPGRADE_FROM_ARCHIVES.each {
dependencies.add(project.dependencies.project(
path: ":distribution:bwc:${unreleasedInfo.gradleProjectName}", configuration: it))
if (upgradeFromVersion.onOrAfter('6.3.0')) {
dependencies.add(project.dependencies.project(
path: ":distribution:bwc:${unreleasedInfo.gradleProjectName}", configuration: "oss-${it}"))
}
}
} else {
UPGRADE_FROM_ARCHIVES.each {
// The version of elasticsearch that we upgrade *from*
dependencies.add("downloads.${it}:elasticsearch:${upgradeFromVersion}@${it}")
if (upgradeFromVersion.onOrAfter('6.3.0')) {
dependencies.add("downloads.${it}:elasticsearch-oss:${upgradeFromVersion}@${it}")
}
}
}

for (Object dependency : dependencies) {
project.dependencies.add(PACKAGING_CONFIGURATION, dependency)
}

project.extensions.esvagrant.upgradeFromVersion = upgradeFromVersion
}

Expand Down
20 changes: 20 additions & 0 deletions buildSrc/src/main/java/org/elasticsearch/gradle/VersionCollection.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@

import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -86,6 +88,7 @@ public class VersionCollection {

private final Version currentVersion;
private final Map<Integer, List<Version>> groupByMajor;
private final Map<Version, UnreleasedVersionInfo> unreleased;

public class UnreleasedVersionInfo {
public final Version version;
Expand Down Expand Up @@ -129,6 +132,16 @@ protected VersionCollection(List<String> versionLines, Version currentVersionPro
assertCurrentVersionMatchesParsed(currentVersionProperty);

assertNoOlderThanTwoMajors();

Map<Version, UnreleasedVersionInfo> unreleased = new HashMap<>();
for (Version unreleasedVersion : getUnreleased()) {
if (unreleasedVersion.equals(currentVersion)) {
continue;
}
unreleased.put(unreleasedVersion,
new UnreleasedVersionInfo(unreleasedVersion, getBranchFor(unreleasedVersion), getGradleProjectNameFor(unreleasedVersion)));
}
this.unreleased = Collections.unmodifiableMap(unreleased);
}

private void assertNoOlderThanTwoMajors() {
Expand All @@ -150,6 +163,13 @@ private void assertCurrentVersionMatchesParsed(Version currentVersionProperty) {
}
}

/**
* Returns info about the unreleased version, or {@code null} if the version is released.
*/
public UnreleasedVersionInfo unreleasedInfo(Version version) {
return unreleased.get(version);
}

public void forPreviousUnreleased(Consumer<UnreleasedVersionInfo> consumer) {
getUnreleased().stream()
.filter(version -> version.equals(currentVersion) == false)
Expand Down
31 changes: 18 additions & 13 deletions distribution/bwc/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,18 +120,19 @@ bwcVersions.forPreviousUnreleased { VersionCollection.UnreleasedVersionInfo unre
}
}

List<File> artifactFiles = []
Map<String, File> artifactFiles = [:]
List<String> projectDirs = []
for (String project : ['zip', 'deb', 'rpm']) {
List<String> projects = ['zip', 'deb', 'rpm']
for (String projectName : projects) {
String baseDir = "distribution"
if (bwcVersion.onOrAfter('6.3.0')) {
baseDir += project == 'zip' ? '/archives' : '/packages'
baseDir += projectName == 'zip' ? '/archives' : '/packages'
// add oss variant first
projectDirs.add("${baseDir}/oss-${project}")
artifactFiles.add(file("${checkoutDir}/${baseDir}/oss-${project}/build/distributions/elasticsearch-oss-${bwcVersion}-SNAPSHOT.${project}"))
projectDirs.add("${baseDir}/oss-${projectName}")
artifactFiles.put("oss-" + projectName, file("${checkoutDir}/${baseDir}/oss-${projectName}/build/distributions/elasticsearch-oss-${bwcVersion}-SNAPSHOT.${projectName}"))
}
projectDirs.add("${baseDir}/${project}")
artifactFiles.add(file("${checkoutDir}/${baseDir}/${project}/build/distributions/elasticsearch-${bwcVersion}-SNAPSHOT.${project}"))
projectDirs.add("${baseDir}/${projectName}")
artifactFiles.put(projectName, file("${checkoutDir}/${baseDir}/${projectName}/build/distributions/elasticsearch-${bwcVersion}-SNAPSHOT.${projectName}"))
}

task buildBwcVersion(type: Exec) {
Expand Down Expand Up @@ -187,7 +188,7 @@ bwcVersions.forPreviousUnreleased { VersionCollection.UnreleasedVersionInfo unre
standardOutput = new IndentingOutputStream(System.out, bwcVersion)
errorOutput = new IndentingOutputStream(System.err, bwcVersion)
doLast {
List missing = artifactFiles.grep { file ->
List missing = artifactFiles.values().grep { file ->
false == file.exists()
}
if (false == missing.empty) {
Expand All @@ -197,12 +198,16 @@ bwcVersions.forPreviousUnreleased { VersionCollection.UnreleasedVersionInfo unre
}
}

artifacts {
for (File artifactFile : artifactFiles) {
String artifactName = artifactFile.name.contains('oss') ? 'elasticsearch-oss' : 'elasticsearch'
for (e in artifactFiles) {
String projectName = e.key
File artifactFile = e.value
String artifactFileName = artifactFile.name
String artifactName = artifactFileName.contains('oss') ? 'elasticsearch-oss' : 'elasticsearch'
String suffix = artifactFile.toString()[-3..-1]
'default' file: artifactFile, name: artifactName, type: suffix, builtBy: buildBwcVersion
}
configurations.create(projectName)
artifacts {
it.add(projectName, [file: artifactFile, name: artifactName, type: suffix, builtBy: buildBwcVersion])
}
}
// make sure no dependencies were added to assemble; we want it to be a no-op
assemble.dependsOn = []
Expand Down

0 comments on commit d9d13f3

Please sign in to comment.