Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Move common repository configuration to java plugin #57057

Merged
merged 10 commits into from May 26, 2020
Merged

Move common repository configuration to java plugin #57057

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
178fcb5
Move common repository configuration to java plugin
rjernst May 21, 2020
be159ec
make public temporarily, and remove dead code
rjernst May 21, 2020
7f0a45c
use gradle secure check
rjernst May 21, 2020
6aaaf5b
iter
rjernst May 21, 2020
d234168
spotless
rjernst May 21, 2020
6084a4d
Merge branch 'master' into buildsplit10
rjernst May 26, 2020
cc839e2
Revert "use gradle secure check"
rjernst May 26, 2020
059255e
add TODO
rjernst May 26, 2020
f66abdf
Merge branch 'master' into buildsplit10
rjernst May 26, 2020
24ea8fe
spotless
rjernst May 26, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
138 changes: 1 addition & 137 deletions buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
View file
Open in desktop
Expand Up @@ -18,7 +18,6 @@
*/
package org.elasticsearch.gradle


import groovy.transform.CompileStatic
import org.apache.commons.io.IOUtils
import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin
Expand All @@ -27,28 +26,10 @@ import org.elasticsearch.gradle.test.ErrorReportingTestListener
import org.elasticsearch.gradle.testclusters.ElasticsearchCluster
import org.elasticsearch.gradle.testclusters.TestClustersPlugin
import org.elasticsearch.gradle.util.GradleUtils
import org.gradle.api.Action
import org.gradle.api.GradleException
import org.gradle.api.InvalidUserDataException
import org.gradle.api.NamedDomainObjectContainer
import org.gradle.api.Plugin
import org.gradle.api.Project
import org.gradle.api.Task
import org.gradle.api.artifacts.Configuration
import org.gradle.api.artifacts.Dependency
import org.gradle.api.artifacts.ModuleDependency
import org.gradle.api.artifacts.ProjectDependency
import org.gradle.api.artifacts.dsl.RepositoryHandler
import org.gradle.api.artifacts.repositories.ExclusiveContentRepository
import org.gradle.api.artifacts.repositories.IvyArtifactRepository
import org.gradle.api.artifacts.repositories.IvyPatternRepositoryLayout
import org.gradle.api.artifacts.repositories.MavenArtifactRepository
import org.gradle.api.credentials.HttpHeaderCredentials
import org.gradle.api.*
import org.gradle.api.execution.TaskActionListener
import org.gradle.api.plugins.ExtraPropertiesExtension
import org.gradle.api.plugins.JavaPlugin
import org.gradle.api.tasks.testing.Test
import org.gradle.authentication.http.HttpHeaderAuthentication
import org.gradle.util.GradleVersion

import java.nio.charset.StandardCharsets
Expand Down Expand Up @@ -90,7 +71,6 @@ class BuildPlugin implements Plugin<Project> {

project.getTasks().register("buildResources", ExportElasticsearchBuildResourcesTask)

configureRepositories(project)
project.extensions.getByType(ExtraPropertiesExtension).set('versions', VersionProperties.versions)
PrecommitTasks.create(project, true)
configureFips140(project)
Expand Down Expand Up @@ -140,122 +120,6 @@ class BuildPlugin implements Plugin<Project> {
}
}

/**
* Makes dependencies non-transitive.
*
* Gradle allows setting all dependencies as non-transitive very easily.
* Sadly this mechanism does not translate into maven pom generation. In order
* to effectively make the pom act as if it has no transitive dependencies,
* we must exclude each transitive dependency of each direct dependency.
*
* Determining the transitive deps of a dependency which has been resolved as
* non-transitive is difficult because the process of resolving removes the
* transitive deps. To sidestep this issue, we create a configuration per
* direct dependency version. This specially named and unique configuration
* will contain all of the transitive dependencies of this particular
* dependency. We can then use this configuration during pom generation
* to iterate the transitive dependencies and add excludes.
*/
static void configureConfigurations(Project project) {
// we want to test compileOnly deps!
project.configurations.getByName(JavaPlugin.TEST_COMPILE_CONFIGURATION_NAME).extendsFrom(project.configurations.getByName(JavaPlugin.COMPILE_ONLY_CONFIGURATION_NAME))

// we are not shipping these jars, we act like dumb consumers of these things
if (project.path.startsWith(':test:fixtures') || project.path == ':build-tools') {
return
}
// fail on any conflicting dependency versions
project.configurations.all({ Configuration configuration ->
if (configuration.name.endsWith('Fixture')) {
// just a self contained test-fixture configuration, likely transitive and hellacious
return
}
configuration.resolutionStrategy {
failOnVersionConflict()
}
})

// force all dependencies added directly to compile/testCompile to be non-transitive, except for ES itself
Closure disableTransitiveDeps = { Dependency dep ->
if (dep instanceof ModuleDependency && !(dep instanceof ProjectDependency)
&& dep.group.startsWith('org.elasticsearch') == false) {
dep.transitive = false
}
}

project.configurations.getByName(JavaPlugin.COMPILE_CONFIGURATION_NAME).dependencies.all(disableTransitiveDeps)
project.configurations.getByName(JavaPlugin.TEST_COMPILE_CONFIGURATION_NAME).dependencies.all(disableTransitiveDeps)
project.configurations.getByName(JavaPlugin.COMPILE_ONLY_CONFIGURATION_NAME).dependencies.all(disableTransitiveDeps)
project.configurations.getByName(JavaPlugin.RUNTIME_ONLY_CONFIGURATION_NAME).dependencies.all(disableTransitiveDeps)
}

/** Adds repositories used by ES dependencies */
static void configureRepositories(Project project) {
project.getRepositories().all { repository ->
if (repository instanceof MavenArtifactRepository) {
final MavenArtifactRepository maven = (MavenArtifactRepository) repository
assertRepositoryURIIsSecure(maven.name, project.path, maven.getUrl())
repository.getArtifactUrls().each { uri -> assertRepositoryURIIsSecure(maven.name, project.path, uri) }
} else if (repository instanceof IvyArtifactRepository) {
final IvyArtifactRepository ivy = (IvyArtifactRepository) repository
assertRepositoryURIIsSecure(ivy.name, project.path, ivy.getUrl())
}
}
RepositoryHandler repos = project.repositories
if (System.getProperty('repos.mavenLocal') != null) {
// with -Drepos.mavenLocal=true we can force checking the local .m2 repo which is
// useful for development ie. bwc tests where we install stuff in the local repository
// such that we don't have to pass hardcoded files to gradle
repos.mavenLocal()
}
repos.jcenter()
repos.ivy { IvyArtifactRepository repo ->
repo.name = 'elasticsearch'
repo.url = 'https://artifacts.elastic.co/downloads'
repo.patternLayout { IvyPatternRepositoryLayout layout ->
layout.artifact 'elasticsearch/[module]-[revision](-[classifier]).[ext]'
}
// this header is not a credential but we hack the capability to send this header to avoid polluting our download stats
repo.credentials(HttpHeaderCredentials, { HttpHeaderCredentials creds ->
creds.name = 'X-Elastic-No-KPI'
creds.value = '1'
} as Action<HttpHeaderCredentials>)
repo.authentication.create('header', HttpHeaderAuthentication)
}
repos.maven { MavenArtifactRepository repo ->
repo.name = 'elastic'
repo.url = 'https://artifacts.elastic.co/maven'
}
String luceneVersion = VersionProperties.lucene
if (luceneVersion.contains('-snapshot')) {
// extract the revision number from the version with a regex matcher
List<String> matches = (luceneVersion =~ /\w+-snapshot-([a-z0-9]+)/).getAt(0) as List<String>
String revision = matches.get(1)
MavenArtifactRepository luceneRepo = repos.maven { MavenArtifactRepository repo ->
repo.name = 'lucene-snapshots'
repo.url = "https://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/${revision}"
}
repos.exclusiveContent { ExclusiveContentRepository exclusiveRepo ->
exclusiveRepo.filter {
it.includeVersionByRegex(/org\.apache\.lucene/, '.*', ".*-snapshot-${revision}")
}
exclusiveRepo.forRepositories(luceneRepo)
}
}
}

static void assertRepositoryURIIsSecure(final String repositoryName, final String projectPath, final URI uri) {
if (uri != null && ["file", "https", "s3"].contains(uri.getScheme()) == false) {
final String message = String.format(
Locale.ROOT,
"repository [%s] on project with path [%s] is not using a secure protocol for artifacts on [%s]",
repositoryName,
projectPath,
uri.toURL())
throw new GradleException(message)
}
}

private static class TestFailureReportingPlugin implements Plugin<Project> {
@Override
void apply(Project project) {
Expand Down
2 changes: 1 addition & 1 deletion buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy
View file
Open in desktop
Expand Up @@ -62,7 +62,7 @@ class StandaloneRestTestPlugin implements Plugin<Project> {
project.pluginManager.apply(TestClustersPlugin)

project.getTasks().create("buildResources", ExportElasticsearchBuildResourcesTask)
BuildPlugin.configureRepositories(project)
ElasticsearchJavaPlugin.configureRepositories(project)
ElasticsearchJavaPlugin.configureTestTasks(project)
ElasticsearchJavaPlugin.configureInputNormalization(project)
BuildPlugin.configureFips140(project)
Expand Down
78 changes: 78 additions & 0 deletions buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchJavaPlugin.java
View file
Open in desktop
Expand Up @@ -36,6 +36,10 @@
import org.gradle.api.artifacts.ModuleDependency;
import org.gradle.api.artifacts.ProjectDependency;
import org.gradle.api.artifacts.ResolutionStrategy;
import org.gradle.api.artifacts.dsl.RepositoryHandler;
import org.gradle.api.artifacts.repositories.ExclusiveContentRepository;
import org.gradle.api.artifacts.repositories.IvyArtifactRepository;
import org.gradle.api.artifacts.repositories.MavenArtifactRepository;
import org.gradle.api.file.FileCollection;
import org.gradle.api.plugins.BasePlugin;
import org.gradle.api.plugins.ExtraPropertiesExtension;
Expand All @@ -56,12 +60,18 @@
import java.io.File;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.net.MalformedURLException;
import java.net.URI;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import static org.elasticsearch.gradle.util.GradleUtils.maybeConfigure;
import static org.elasticsearch.gradle.util.Util.toStringable;
Expand All @@ -77,6 +87,7 @@ public void apply(Project project) {

project.getPluginManager().apply(JavaPlugin.class);
configureConfigurations(project);
configureRepositories(project);
configureCompile(project);
configureInputNormalization(project);
configureTestTasks(project);
Expand Down Expand Up @@ -137,6 +148,73 @@ public static void configureConfigurations(Project project) {
disableTransitiveDeps.accept(JavaPlugin.RUNTIME_ONLY_CONFIGURATION_NAME);
}

private static final Pattern LUCENE_SNAPSHOT_REGEX = Pattern.compile("\\w+-snapshot-([a-z0-9]+)");

/** Adds repositories used by ES dependencies */
public static void configureRepositories(Project project) {
// ensure all repositories use secure urls
project.getRepositories().all(repository -> {
if (repository instanceof MavenArtifactRepository) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we can now replace all of this with the following which was introduced in Gradle 6.0.

if (repository instanceof UrlArtifactRepository) {
    ((UrlArtifactRepository) repository).setAllowInsecureProtocol(false);
}

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was not able to do this because the gradle setting only controls a deprecation warning, it does not fail the build.

final MavenArtifactRepository maven = (MavenArtifactRepository) repository;
assertRepositoryURIIsSecure(maven.getName(), project.getPath(), maven.getUrl());
for (URI uri : maven.getArtifactUrls()) {
assertRepositoryURIIsSecure(maven.getName(), project.getPath(), uri);
}
} else if (repository instanceof IvyArtifactRepository) {
final IvyArtifactRepository ivy = (IvyArtifactRepository) repository;
assertRepositoryURIIsSecure(ivy.getName(), project.getPath(), ivy.getUrl());
}
});
RepositoryHandler repos = project.getRepositories();
if (System.getProperty("repos.mavenLocal") != null) {
// with -Drepos.mavenLocal=true we can force checking the local .m2 repo which is
// useful for development ie. bwc tests where we install stuff in the local repository
// such that we don't have to pass hardcoded files to gradle
repos.mavenLocal();
}
repos.jcenter();

String luceneVersion = VersionProperties.getLucene();
if (luceneVersion.contains("-snapshot")) {
// extract the revision number from the version with a regex matcher
Matcher matcher = LUCENE_SNAPSHOT_REGEX.matcher(luceneVersion);
if (matcher.find() == false) {
throw new GradleException("Malformed lucene snapshot version: " + luceneVersion);
}
String revision = matcher.group(1);
MavenArtifactRepository luceneRepo = repos.maven(repo -> {
repo.setName("lucene-snapshots");
repo.setUrl("https://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/" + revision);
});
repos.exclusiveContent(exclusiveRepo -> {
exclusiveRepo.filter(descriptor -> {
descriptor.includeVersionByRegex("org\\.apache\\.lucene", ".*", ".*-snapshot-" + revision);
});
exclusiveRepo.forRepositories(luceneRepo);
});
}
}

private static final List<String> SECURE_URL_SCHEMES = Arrays.asList("file", "https", "s3");

private static void assertRepositoryURIIsSecure(final String repositoryName, final String projectPath, final URI uri) {
if (uri != null && SECURE_URL_SCHEMES.contains(uri.getScheme()) == false) {
String url;
try {
url = uri.toURL().toString();
} catch (MalformedURLException e) {
throw new IllegalStateException(e);
}
final String message = String.format(
Locale.ROOT,
"repository [%s] on project with path [%s] is not using a secure protocol for artifacts on [%s]",
repositoryName,
projectPath,
url);
throw new GradleException(message);
}
}

/** Adds compiler settings to the project */
public static void configureCompile(Project project) {
project.getExtensions().getExtraProperties().set("compactProfile", "full");
Expand Down