diff --git a/build.gradle b/build.gradle index 7630c37c1e1d2..c7569a73c5301 100644 --- a/build.gradle +++ b/build.gradle @@ -482,7 +482,7 @@ gradle.projectsEvaluated { } -/* Remove assemble on all qa projects because we don't need to publish +/* Remove assemble/dependenciesInfo on all qa projects because we don't need to publish * artifacts for them. */ gradle.projectsEvaluated { subprojects { @@ -492,6 +492,11 @@ gradle.projectsEvaluated { project.tasks.remove(assemble) project.build.dependsOn.remove('assemble') } + Task dependenciesInfo = project.tasks.findByName('dependenciesInfo') + if (dependenciesInfo) { + project.tasks.remove(dependenciesInfo) + project.precommit.dependsOn.remove('dependenciesInfo') + } } } } diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index a44b9c849d333..d9fc9118e1d48 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -746,6 +746,10 @@ class BuildPlugin implements Plugin { private static configureDependenciesInfo(Project project) { Task deps = project.tasks.create("dependenciesInfo", DependenciesInfoTask.class) - deps.dependencies = project.configurations.compile.allDependencies + deps.runtimeConfiguration = project.configurations.runtime + deps.compileOnlyConfiguration = project.configurations.compileOnly + project.afterEvaluate { + deps.mappings = project.dependencyLicenses.mappings + } } } diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/DependenciesInfoTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/DependenciesInfoTask.groovy index b42e6cc8e3caa..e62fe4db954c5 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/DependenciesInfoTask.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/DependenciesInfoTask.groovy @@ -19,14 +19,19 @@ package org.elasticsearch.gradle +import org.elasticsearch.gradle.precommit.DependencyLicensesTask import org.gradle.api.DefaultTask +import org.gradle.api.artifacts.Configuration import org.gradle.api.artifacts.Dependency +import org.gradle.api.artifacts.DependencyResolutionListener import org.gradle.api.artifacts.DependencySet import org.gradle.api.tasks.Input import org.gradle.api.tasks.InputDirectory import org.gradle.api.tasks.OutputFile import org.gradle.api.tasks.TaskAction +import java.util.regex.Matcher +import java.util.regex.Pattern /** * A task to gather information about the dependencies and export them into a csv file. @@ -44,7 +49,14 @@ public class DependenciesInfoTask extends DefaultTask { /** Dependencies to gather information from. */ @Input - public DependencySet dependencies + public Configuration runtimeConfiguration + + /** We subtract compile-only dependencies. */ + @Input + public Configuration compileOnlyConfiguration + + @Input + public LinkedHashMap mappings /** Directory to read license files */ @InputDirectory @@ -59,15 +71,34 @@ public class DependenciesInfoTask extends DefaultTask { @TaskAction public void generateDependenciesInfo() { + + final DependencySet runtimeDependencies = runtimeConfiguration.getAllDependencies() + // we have to resolve the transitive dependencies and create a group:artifactId:version map + final Set compileOnlyArtifacts = + compileOnlyConfiguration + .getResolvedConfiguration() + .resolvedArtifacts + .collect { it -> "${it.moduleVersion.id.group}:${it.moduleVersion.id.name}:${it.moduleVersion.id.version}" } + final StringBuilder output = new StringBuilder() - for (Dependency dependency : dependencies) { - // Only external dependencies are checked - if (dependency.group != null && dependency.group.contains("elasticsearch") == false) { - final String url = createURL(dependency.group, dependency.name, dependency.version) - final String licenseType = getLicenseType(dependency.group, dependency.name) - output.append("${dependency.group}:${dependency.name},${dependency.version},${url},${licenseType}\n") + for (final Dependency dependency : runtimeDependencies) { + // we do not need compile-only dependencies here + if (compileOnlyArtifacts.contains("${dependency.group}:${dependency.name}:${dependency.version}")) { + continue + } + // only external dependencies are checked + if (dependency.group != null && dependency.group.contains("org.elasticsearch")) { + continue } + + final String url = createURL(dependency.group, dependency.name, dependency.version) + final String dependencyName = DependencyLicensesTask.getDependencyName(mappings, dependency.name) + logger.info("mapped dependency ${dependency.group}:${dependency.name} to ${dependencyName} for license info") + + final String licenseType = getLicenseType(dependency.group, dependencyName) + output.append("${dependency.group}:${dependency.name},${dependency.version},${url},${licenseType}\n") + } outputFile.setText(output.toString(), 'UTF-8') } @@ -173,7 +204,7 @@ are met: derived from this software without specific prior written permission\\.| (3\\.)? Neither the name of .+ nor the names of its contributors may be used to endorse or promote products derived from - this software without specific prior written permission\\.) + this software without specific prior written permission\\.) THIS SOFTWARE IS PROVIDED BY .+ (``|''|")AS IS(''|") AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/DependencyLicensesTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/DependencyLicensesTask.groovy index 4d292d87ec39c..b2854257951a8 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/DependencyLicensesTask.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/DependencyLicensesTask.groovy @@ -109,6 +109,10 @@ public class DependencyLicensesTask extends DefaultTask { mappings.put(from, to) } + public LinkedHashMap getMappings() { + return new LinkedHashMap<>(mappings) + } + /** * Add a rule which will skip SHA checking for the given dependency name. This should be used for * locally build dependencies, which cause the sha to change constantly. @@ -129,10 +133,6 @@ public class DependencyLicensesTask extends DefaultTask { throw new GradleException("Licences dir ${licensesDir} does not exist, but there are dependencies") } - - // order is the same for keys and values iteration since we use a linked hashmap - List mapped = new ArrayList<>(mappings.values()) - Pattern mappingsPattern = Pattern.compile('(' + mappings.keySet().join(')|(') + ')') Map licenses = new HashMap<>() Map notices = new HashMap<>() Set shaFiles = new HashSet() @@ -162,16 +162,10 @@ public class DependencyLicensesTask extends DefaultTask { checkSha(dependency, jarName, shaFiles) } - logger.info("Checking license/notice for " + depName) - Matcher match = mappingsPattern.matcher(depName) - if (match.matches()) { - int i = 0 - while (i < match.groupCount() && match.group(i + 1) == null) ++i; - logger.info("Mapped dependency name ${depName} to ${mapped.get(i)} for license check") - depName = mapped.get(i) - } - checkFile(depName, jarName, licenses, 'LICENSE') - checkFile(depName, jarName, notices, 'NOTICE') + final String dependencyName = getDependencyName(mappings, depName) + logger.info("mapped dependency name ${depName} to ${dependencyName} for license/notice check") + checkFile(dependencyName, jarName, licenses, 'LICENSE') + checkFile(dependencyName, jarName, notices, 'NOTICE') } licenses.each { license, count -> @@ -189,6 +183,19 @@ public class DependencyLicensesTask extends DefaultTask { } } + public static String getDependencyName(final LinkedHashMap mappings, final String dependencyName) { + // order is the same for keys and values iteration since we use a linked hashmap + List mapped = new ArrayList<>(mappings.values()) + Pattern mappingsPattern = Pattern.compile('(' + mappings.keySet().join(')|(') + ')') + Matcher match = mappingsPattern.matcher(dependencyName) + if (match.matches()) { + int i = 0 + while (i < match.groupCount() && match.group(i + 1) == null) ++i; + return mapped.get(i) + } + return dependencyName + } + private File getShaFile(String jarName) { return new File(licensesDir, jarName + SHA_EXTENSION) } diff --git a/client/client-benchmark-noop-api-plugin/build.gradle b/client/client-benchmark-noop-api-plugin/build.gradle index bee41034c3ce5..cc84207d90d22 100644 --- a/client/client-benchmark-noop-api-plugin/build.gradle +++ b/client/client-benchmark-noop-api-plugin/build.gradle @@ -31,6 +31,9 @@ esplugin { tasks.remove(assemble) build.dependsOn.remove('assemble') +dependencyLicenses.enabled = false +dependenciesInfo.enabled = false + compileJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-unchecked" // no unit tests diff --git a/distribution/build.gradle b/distribution/build.gradle index 23d1d7b66ad43..2d3c8d9eb35fd 100644 --- a/distribution/build.gradle +++ b/distribution/build.gradle @@ -32,6 +32,11 @@ Collection distributions = project('archives').subprojects + project('packages') // Concatenates the dependencies CSV files into a single file task generateDependenciesReport(type: ConcatFilesTask) { + project.rootProject.allprojects { + afterEvaluate { + if (it.tasks.findByName("dependenciesInfo")) dependsOn it.tasks.dependenciesInfo + } + } files = fileTree(dir: project.rootDir, include: '**/dependencies.csv' ) headerLine = "name,version,url,license" target = new File(System.getProperty('csv')?: "${project.buildDir}/dependencies/es-dependencies.csv") diff --git a/test/fixtures/example-fixture/build.gradle b/test/fixtures/example-fixture/build.gradle index 225a2cf9deba6..ce562e89abb7f 100644 --- a/test/fixtures/example-fixture/build.gradle +++ b/test/fixtures/example-fixture/build.gradle @@ -22,3 +22,5 @@ test.enabled = false // Not published so no need to assemble tasks.remove(assemble) build.dependsOn.remove('assemble') + +dependenciesInfo.enabled = false diff --git a/x-pack/qa/build.gradle b/x-pack/qa/build.gradle index 1570b218592fe..24b6618b7d8f6 100644 --- a/x-pack/qa/build.gradle +++ b/x-pack/qa/build.gradle @@ -28,5 +28,9 @@ gradle.projectsEvaluated { project.tasks.remove(assemble) project.build.dependsOn.remove('assemble') } + Task dependenciesInfo = project.tasks.findByName('dependenciesInfo') + if (dependenciesInfo) { + project.precommit.dependsOn.remove('dependenciesInfo') + } } } diff --git a/x-pack/qa/sql/build.gradle b/x-pack/qa/sql/build.gradle index 18ad4067805a6..7079c1e3fd231 100644 --- a/x-pack/qa/sql/build.gradle +++ b/x-pack/qa/sql/build.gradle @@ -22,6 +22,7 @@ dependencies { test.enabled = false dependencyLicenses.enabled = false +dependenciesInfo.enabled = false // the main files are actually test files, so use the appropriate forbidden api sigs forbiddenApisMain { diff --git a/x-pack/test/feature-aware/build.gradle b/x-pack/test/feature-aware/build.gradle index 217ed25a2d4b1..11b0e67183c8f 100644 --- a/x-pack/test/feature-aware/build.gradle +++ b/x-pack/test/feature-aware/build.gradle @@ -10,6 +10,7 @@ dependencies { forbiddenApisMain.enabled = true dependencyLicenses.enabled = false +dependenciesInfo.enabled = false jarHell.enabled = false