From a971622d87cf4f2e54ac35c2bf9cf1648e984372 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Wed, 10 Mar 2021 16:39:43 +0100 Subject: [PATCH 1/9] Port more plugins from groovy to java - In general we want to move all production build logic to java - this is an incremental step towards that goal. --- .../elasticsearch/gradle/NoticeTask.groovy | 169 --------- .../gradle/plugin/PluginBuildPlugin.groovy | 272 -------------- .../test/TestWithDependenciesPlugin.groovy | 58 --- .../org/elasticsearch/gradle/NoticeTask.java | 203 +++++++++++ .../gradle/plugin/PluginBuildPlugin.java | 332 ++++++++++++++++++ .../test/TestWithDependenciesPlugin.java | 70 ++++ 6 files changed, 605 insertions(+), 499 deletions(-) delete mode 100644 buildSrc/src/main/groovy/org/elasticsearch/gradle/NoticeTask.groovy delete mode 100644 buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy delete mode 100644 buildSrc/src/main/groovy/org/elasticsearch/gradle/test/TestWithDependenciesPlugin.groovy create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/NoticeTask.java create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/plugin/PluginBuildPlugin.java create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/test/TestWithDependenciesPlugin.java diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/NoticeTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/NoticeTask.groovy deleted file mode 100644 index dfd1a2353a514..0000000000000 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/NoticeTask.groovy +++ /dev/null @@ -1,169 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.gradle - -import org.gradle.api.DefaultTask -import org.gradle.api.file.FileCollection -import org.gradle.api.file.FileTree -import org.gradle.api.file.SourceDirectorySet -import org.gradle.api.tasks.InputFile -import org.gradle.api.tasks.InputFiles -import org.gradle.api.tasks.Optional -import org.gradle.api.tasks.OutputFile -import org.gradle.api.tasks.TaskAction - -/** - * A task to create a notice file which includes dependencies' notices. - */ -class NoticeTask extends DefaultTask { - - @InputFile - File inputFile = project.rootProject.file('NOTICE.txt') - - @OutputFile - File outputFile = new File(project.buildDir, "notices/${name}/NOTICE.txt") - - private FileTree sources - - /** Directories to include notices from */ - private List licensesDirs = new ArrayList<>() - - NoticeTask() { - description = 'Create a notice file from dependencies' - // Default licenses directory is ${projectDir}/licenses (if it exists) - File licensesDir = new File(project.projectDir, 'licenses') - if (licensesDir.exists()) { - licensesDirs.add(licensesDir) - } - } - - /** Add notices from the specified directory. */ - void licensesDir(File licensesDir) { - licensesDirs.add(licensesDir) - } - - void source(Object source) { - if (sources == null) { - sources = project.fileTree(source) - } else { - sources += project.fileTree(source) - } - } - - void source(SourceDirectorySet source) { - if (sources == null) { - sources = source - } else { - sources += source - } - } - - @TaskAction - void generateNotice() { - StringBuilder output = new StringBuilder() - output.append(inputFile.getText('UTF-8')) - output.append('\n\n') - // This is a map rather than a set so that the sort order is the 3rd - // party component names, unaffected by the full path to the various files - Map seen = new TreeMap<>() - noticeFiles.each { File file -> - String name = file.name.replaceFirst(/-NOTICE\.txt$/, "") - if (seen.containsKey(name)) { - File prevFile = seen.get(name) - if (prevFile.text != file.text) { - throw new RuntimeException("Two different notices exist for dependency '" + - name + "': " + prevFile + " and " + file) - } - } else { - seen.put(name, file) - } - } - - // Add all LICENSE and NOTICE files in licenses directory - for (Map.Entry entry : seen.entrySet()) { - String name = entry.getKey() - File file = entry.getValue() - appendFile(file, name, 'NOTICE', output) - appendFile(new File(file.parentFile, "${name}-LICENSE.txt"), name, 'LICENSE', output) - } - - // Find any source files with "@notice" annotated license header - for (File sourceFile : sources.files) { - boolean isPackageInfo = sourceFile.name == 'package-info.java' - boolean foundNotice = false - boolean inNotice = false - StringBuilder header = new StringBuilder() - String packageDeclaration - - for (String line : sourceFile.readLines()) { - if (isPackageInfo && packageDeclaration == null && line.startsWith('package')) { - packageDeclaration = line - } - - if (foundNotice == false) { - foundNotice = line.contains('@notice') - inNotice = true - } else { - if (line.contains('*/')) { - inNotice = false - - if (isPackageInfo == false) { - break - } - } else if (inNotice) { - header.append(line.stripMargin('*')) - header.append('\n') - } - } - } - - if (foundNotice) { - appendText(header.toString(), isPackageInfo ? packageDeclaration : sourceFile.name, '', output) - } - } - outputFile.setText(output.toString(), 'UTF-8') - } - - @InputFiles - @Optional - FileCollection getNoticeFiles() { - FileTree tree - licensesDirs.each { dir -> - if (tree == null) { - tree = project.fileTree(dir) - } else { - tree += project.fileTree(dir) - } - } - - return tree?.matching { include '**/*-NOTICE.txt' } - } - - @InputFiles - @Optional - FileCollection getSources() { - return sources - } - - static void appendFile(File file, String name, String type, StringBuilder output) { - String text = file.getText('UTF-8') - if (text.trim().isEmpty()) { - return - } - appendText(text, name, type, output) - } - - static void appendText(String text, String name, String type, StringBuilder output) { - output.append('================================================================================\n') - output.append("${name} ${type}\n") - output.append('================================================================================\n') - output.append(text) - output.append('\n\n') - } -} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy deleted file mode 100644 index f9f3166dda661..0000000000000 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy +++ /dev/null @@ -1,272 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ -package org.elasticsearch.gradle.plugin - -import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin -import org.elasticsearch.gradle.BuildPlugin -import org.elasticsearch.gradle.NoticeTask -import org.elasticsearch.gradle.Version -import org.elasticsearch.gradle.VersionProperties -import org.elasticsearch.gradle.dependencies.CompileOnlyResolvePlugin -import org.elasticsearch.gradle.info.BuildParams -import org.elasticsearch.gradle.test.RestTestBasePlugin -import org.elasticsearch.gradle.testclusters.ElasticsearchCluster -import org.elasticsearch.gradle.testclusters.RunTask -import org.elasticsearch.gradle.testclusters.TestClustersPlugin -import org.elasticsearch.gradle.util.GradleUtils -import org.elasticsearch.gradle.util.Util -import org.gradle.api.InvalidUserDataException -import org.gradle.api.NamedDomainObjectContainer -import org.gradle.api.Plugin -import org.gradle.api.Project -import org.gradle.api.Task -import org.gradle.api.plugins.BasePlugin -import org.gradle.api.publish.maven.MavenPublication -import org.gradle.api.publish.maven.plugins.MavenPublishPlugin -import org.gradle.api.tasks.Copy -import org.gradle.api.tasks.SourceSet -import org.gradle.api.tasks.TaskProvider -import org.gradle.api.tasks.bundling.Zip - -/** - * Encapsulates build configuration for an Elasticsearch plugin. - */ -class PluginBuildPlugin implements Plugin { - - public static final String PLUGIN_EXTENSION_NAME = 'esplugin' - - @Override - void apply(Project project) { - project.pluginManager.apply(BuildPlugin) - project.pluginManager.apply(RestTestBasePlugin) - project.pluginManager.apply(CompileOnlyResolvePlugin.class) - - PluginPropertiesExtension extension = project.extensions.create(PLUGIN_EXTENSION_NAME, PluginPropertiesExtension, project) - configureDependencies(project) - - TaskProvider bundleTask = createBundleTasks(project, extension) - - project.afterEvaluate { - project.extensions.getByType(PluginPropertiesExtension).extendedPlugins.each { pluginName -> - // Auto add dependent modules to the test cluster - if (project.findProject(":modules:${pluginName}") != null) { - project.testClusters.all { - module(":modules:${pluginName}") - } - } - } - PluginPropertiesExtension extension1 = project.getExtensions().getByType(PluginPropertiesExtension.class) - configurePublishing(project, extension1) - String name = extension1.name - project.archivesBaseName = name - project.description = extension1.description - - if (extension1.name == null) { - throw new InvalidUserDataException('name is a required setting for esplugin') - } - if (extension1.description == null) { - throw new InvalidUserDataException('description is a required setting for esplugin') - } - if (extension1.type != PluginType.BOOTSTRAP && extension1.classname == null) { - throw new InvalidUserDataException('classname is a required setting for esplugin') - } - - Map properties = [ - 'name' : extension1.name, - 'description' : extension1.description, - 'version' : extension1.version, - 'elasticsearchVersion': Version.fromString(VersionProperties.elasticsearch).toString(), - 'javaVersion' : project.targetCompatibility as String, - 'classname' : extension1.type == PluginType.BOOTSTRAP ? "" : extension1.classname, - 'extendedPlugins' : extension1.extendedPlugins.join(','), - 'hasNativeController' : extension1.hasNativeController, - 'requiresKeystore' : extension1.requiresKeystore, - 'type' : extension1.type.toString(), - 'javaOpts' : extension1.javaOpts, - 'licensed' : extension1.licensed, - ] - project.tasks.named('pluginProperties').configure { - expand(properties) - inputs.properties(properties) - } - BuildParams.withInternalBuild { - boolean isModule = GradleUtils.isModuleProject(project.path) - boolean isXPackModule = isModule && project.path.startsWith(":x-pack") - if (isModule == false || isXPackModule) { - addNoticeGeneration(project, extension1) - } - } - } - - BuildParams.withInternalBuild { - // We've ported this from multiple build scripts where we see this pattern into - // an extension method as a first step of consolidation. - // We might want to port this into a general pattern later on. - project.ext.addQaCheckDependencies = { - project.afterEvaluate { - // let check depend on check tasks of qa sub-projects - def checkTaskProvider = project.tasks.named("check") - def qaSubproject = project.subprojects.find { it.path == project.path + ":qa" } - if(qaSubproject) { - qaSubproject.subprojects.each {p -> - checkTaskProvider.configure {it.dependsOn(p.path + ":check") } - } - } - } - } - - project.tasks.named('testingConventions').configure { - naming.clear() - naming { - Tests { - baseClass 'org.apache.lucene.util.LuceneTestCase' - } - IT { - baseClass 'org.elasticsearch.test.ESIntegTestCase' - baseClass 'org.elasticsearch.test.rest.ESRestTestCase' - baseClass 'org.elasticsearch.test.ESSingleNodeTestCase' - } - } - } - } - - project.configurations.getByName('default') - .extendsFrom(project.configurations.getByName('runtimeClasspath')) - - // allow running ES with this plugin in the foreground of a build - NamedDomainObjectContainer testClusters = project.extensions.getByName(TestClustersPlugin.EXTENSION_NAME) - ElasticsearchCluster runCluster = testClusters.create('runTask') { ElasticsearchCluster cluster -> - if (GradleUtils.isModuleProject(project.path)) { - cluster.module(bundleTask.flatMap { t -> t.getArchiveFile() }) - } else { - cluster.plugin(bundleTask.flatMap { t -> t.getArchiveFile() }) - } - } - - project.tasks.register('run', RunTask) { - useCluster(runCluster) - dependsOn(project.tasks.named("bundlePlugin")) - } - } - - private static void configurePublishing(Project project, PluginPropertiesExtension extension) { - if (project.plugins.hasPlugin(MavenPublishPlugin)) { - project.publishing.publications.elastic(MavenPublication).artifactId(extension.name) - } - } - - private static void configureDependencies(Project project) { - project.dependencies { - if (BuildParams.internal) { - compileOnly project.project(':server') - testImplementation project.project(':test:framework') - } else { - compileOnly "org.elasticsearch:elasticsearch:${project.versions.elasticsearch}" - testImplementation "org.elasticsearch.test:framework:${project.versions.elasticsearch}" - } - // we "upgrade" these optional deps to provided for plugins, since they will run - // with a full elasticsearch server that includes optional deps - compileOnly "org.locationtech.spatial4j:spatial4j:${project.versions.spatial4j}" - compileOnly "org.locationtech.jts:jts-core:${project.versions.jts}" - compileOnly "org.apache.logging.log4j:log4j-api:${project.versions.log4j}" - compileOnly "org.apache.logging.log4j:log4j-core:${project.versions.log4j}" - compileOnly "org.elasticsearch:jna:${project.versions.jna}" - } - } - - /** - * Adds a bundlePlugin task which builds the zip containing the plugin jars, - * metadata, properties, and packaging files - */ - private static TaskProvider createBundleTasks(Project project, PluginPropertiesExtension extension) { - File pluginMetadata = project.file('src/main/plugin-metadata') - File templateFile = new File(project.buildDir, "templates/plugin-descriptor.properties") - - // create tasks to build the properties file for this plugin - TaskProvider copyPluginPropertiesTemplate = project.tasks.register('copyPluginPropertiesTemplate') { - outputs.file(templateFile) - doLast { - InputStream resourceTemplate = PluginBuildPlugin.getResourceAsStream("/${templateFile.name}") - templateFile.setText(resourceTemplate.getText('UTF-8'), 'UTF-8') - } - } - - TaskProvider buildProperties = project.tasks.register('pluginProperties', Copy) { - dependsOn(copyPluginPropertiesTemplate) - from(templateFile) - into("${project.buildDir}/generated-resources") - } - - // add the plugin properties and metadata to test resources, so unit tests can - // know about the plugin (used by test security code to statically initialize the plugin in unit tests) - SourceSet testSourceSet = project.sourceSets.test - testSourceSet.output.dir("${project.buildDir}/generated-resources", builtBy: buildProperties) - testSourceSet.resources.srcDir(pluginMetadata) - - // create the actual bundle task, which zips up all the files for the plugin - TaskProvider bundle = project.tasks.register('bundlePlugin', Zip) { - from buildProperties - from(pluginMetadata) { - // metadata (eg custom security policy) - // the codebases properties file is only for tests and not needed in production - exclude 'plugin-security.codebases' - } - /* - * If the plugin is using the shadow plugin then we need to bundle - * that shadow jar. - */ - from { project.plugins.hasPlugin(ShadowPlugin) ? project.shadowJar : project.jar } - from project.configurations.runtimeClasspath - project.configurations.getByName( - CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME - ) - // extra files for the plugin to go into the zip - from('src/main/packaging') // TODO: move all config/bin/_size/etc into packaging - from('src/main') { - include 'config/**' - include 'bin/**' - } - } - project.tasks.named(BasePlugin.ASSEMBLE_TASK_NAME).configure { - dependsOn(bundle) - } - - // also make the zip available as a configuration (used when depending on this project) - project.configurations.create('zip') - project.artifacts.add('zip', bundle) - - return bundle - } - - /** Configure the pom for the main jar of this plugin */ - - protected static void addNoticeGeneration(Project project, PluginPropertiesExtension extension) { - File licenseFile = extension.licenseFile - if (licenseFile != null) { - project.tasks.named('bundlePlugin').configure { - from(licenseFile.parentFile) { - include(licenseFile.name) - rename { 'LICENSE.txt' } - } - } - } - File noticeFile = extension.noticeFile - if (noticeFile != null) { - TaskProvider generateNotice = project.tasks.register('generateNotice', NoticeTask) { - inputFile = noticeFile - source(Util.getJavaMainSourceSet(project).get().allJava) - } - project.tasks.named('bundlePlugin').configure { - from(generateNotice) - } - } - } - - static boolean isModuleProject(String projectPath) { - return projectPath.contains("modules:") || projectPath.startsWith(":x-pack:plugin") || projectPath.path.startsWith(':x-pack:quota-aware-fs') - } -} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/TestWithDependenciesPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/TestWithDependenciesPlugin.groovy deleted file mode 100644 index f9ef9a0da1070..0000000000000 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/TestWithDependenciesPlugin.groovy +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.gradle.test - -import org.elasticsearch.gradle.plugin.PluginBuildPlugin -import org.gradle.api.Plugin -import org.gradle.api.Project -import org.gradle.api.artifacts.Dependency -import org.gradle.api.artifacts.ProjectDependency -import org.gradle.api.tasks.Copy - -/** - * A plugin to run tests that depend on other plugins or modules. - * - * This plugin will add the plugin-metadata and properties files for each - * dependency to the test source set. - */ -class TestWithDependenciesPlugin implements Plugin { - - @Override - void apply(Project project) { - if (project.isEclipse) { - /* The changes this plugin makes both break and aren't needed by - * Eclipse. This is because Eclipse flattens main and test - * dependencies into a single dependency. Because Eclipse is - * "special".... */ - return - } - - project.configurations.testImplementation.dependencies.all { Dependency dep -> - // this closure is run every time a compile dependency is added - if (dep instanceof ProjectDependency && dep.dependencyProject.plugins.hasPlugin(PluginBuildPlugin)) { - project.gradle.projectsEvaluated { - addPluginResources(project, dep.dependencyProject) - } - } - } - } - - private static addPluginResources(Project project, Project pluginProject) { - String outputDir = "${project.buildDir}/generated-resources/${pluginProject.name}" - String camelName = pluginProject.name.replaceAll(/-(\w)/) { _, c -> c.toUpperCase(Locale.ROOT) } - String taskName = "copy" + camelName[0].toUpperCase(Locale.ROOT) + camelName.substring(1) + "Metadata" - project.tasks.register(taskName, Copy.class) { - into(outputDir) - from(pluginProject.tasks.pluginProperties) - from(pluginProject.file('src/main/plugin-metadata')) - } - - project.sourceSets.test.output.dir(outputDir, builtBy: taskName) - } -} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/NoticeTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/NoticeTask.java new file mode 100644 index 0000000000000..f54a89769529c --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/NoticeTask.java @@ -0,0 +1,203 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle; + +import groovy.lang.Reference; +import org.apache.commons.io.FileUtils; +import org.codehaus.groovy.runtime.ResourceGroovyMethods; +import org.codehaus.groovy.runtime.StringGroovyMethods; +import org.gradle.api.DefaultTask; +import org.gradle.api.file.FileCollection; +import org.gradle.api.file.FileTree; +import org.gradle.api.file.SourceDirectorySet; +import org.gradle.api.tasks.InputFile; +import org.gradle.api.tasks.InputFiles; +import org.gradle.api.tasks.Optional; +import org.gradle.api.tasks.OutputFile; +import org.gradle.api.tasks.TaskAction; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +import static org.apache.commons.io.FileUtils.readFileToString; + +/** + * A task to create a notice file which includes dependencies' notices. + */ +public class NoticeTask extends DefaultTask { + + @InputFile + private File inputFile = getProject().getRootProject().file("NOTICE.txt"); + @OutputFile + private File outputFile = new File(getProject().getBuildDir(), "notices/" + getName() + "/NOTICE.txt"); + private FileTree sources; + /** + * Directories to include notices from + */ + private List licensesDirs = new ArrayList(); + + public NoticeTask() { + setDescription("Create a notice file from dependencies"); + // Default licenses directory is ${projectDir}/licenses (if it exists) + File licensesDir = new File(getProject().getProjectDir(), "licenses"); + if (licensesDir.exists()) { + licensesDirs.add(licensesDir); + } + } + + /** + * Add notices from the specified directory. + */ + public void licensesDir(File licensesDir) { + licensesDirs.add(licensesDir); + } + + public void source(Object source) { + if (sources == null) { + sources = getProject().fileTree(source); + } else { + sources = sources.plus(getProject().fileTree(source)); + } + + } + + public void source(SourceDirectorySet source) { + if (sources == null) { + sources = source; + } else { + sources = sources.plus(source); + } + + } + + @TaskAction + public void generateNotice() throws IOException { + StringBuilder output = new StringBuilder(); + output.append(readFileToString(inputFile, "UTF-8")); + output.append("\n\n"); + // This is a map rather than a set so that the sort order is the 3rd + // party component names, unaffected by the full path to the various files + final Map seen = new TreeMap(); + for (File file : getNoticeFiles()) { + String name = file.getName().replaceFirst(" -NOTICE\\.txt$ ", ""); + if (seen.containsKey(name)) { + File prevFile = seen.get(name); + String previousFileText = readFileToString(prevFile, "UTF-8"); + if (previousFileText.equals(readFileToString(file, "UTF-8")) == false) { + throw new RuntimeException("Two different notices exist for dependency '" + name + "': " + prevFile + " and " + file); + } + } else { + seen.put(name, file); + } + } + + // Add all LICENSE and NOTICE files in licenses directory + for (Map.Entry entry : seen.entrySet()) { + final String name = entry.getKey(); + File file = entry.getValue(); + appendFile(file, name, "NOTICE", output); + appendFile(new File(file.getParentFile(), name + "-LICENSE.txt"), name, "LICENSE", output); + } + + // Find any source files with "@notice" annotated license header + for (File sourceFile : sources.getFiles()) { + boolean isPackageInfo = sourceFile.getName().equals("package-info.java"); + boolean foundNotice = false; + boolean inNotice = false; + StringBuilder header = new StringBuilder(); + String packageDeclaration = null; + + for (String line : FileUtils.readLines(sourceFile)) { + if (isPackageInfo && packageDeclaration == null && line.startsWith("package")) { + packageDeclaration = line; + } + + if (foundNotice == false) { + foundNotice = line.contains("@notice"); + inNotice = true; + } else { + if (line.contains("*/")) { + inNotice = false; + + if (isPackageInfo == false) { + break; + } + + } else if (inNotice) { + header.append(StringGroovyMethods.stripMargin(line, "*")); + header.append("\n"); + } + } + } + + if (foundNotice) { + appendText(header.toString(), isPackageInfo ? packageDeclaration : sourceFile.getName(), "", output); + } + } + + FileUtils.write(outputFile, output.toString(), "UTF-8"); + } + + @InputFiles + @Optional + public FileCollection getNoticeFiles() { + final Reference tree = new Reference<>(); + licensesDirs.forEach(dir -> { + if (tree.get() == null) { + tree.set(getProject().fileTree(dir)); + } else { + tree.set(tree.get().plus(getProject().fileTree(dir))); + } + }); + return tree.get().matching(patternFilterable -> patternFilterable.include("**/*-NOTICE.txt")); + } + + @InputFiles + @Optional + public FileCollection getSources() { + return sources; + } + + public static void appendFile(File file, String name, String type, StringBuilder output) throws IOException { + String text = ResourceGroovyMethods.getText(file, "UTF-8"); + if (text.trim().isEmpty()) { + return; + } + appendText(text, name, type, output); + } + + public static void appendText(String text, final String name, final String type, StringBuilder output) { + output.append("================================================================================\n"); + output.append(name + " " + type + "\n"); + output.append("================================================================================\n"); + output.append(text); + output.append("\n\n"); + } + + public File getInputFile() { + return inputFile; + } + + public void setInputFile(File inputFile) { + this.inputFile = inputFile; + } + + public File getOutputFile() { + return outputFile; + } + + public void setOutputFile(File outputFile) { + this.outputFile = outputFile; + } + +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/plugin/PluginBuildPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/plugin/PluginBuildPlugin.java new file mode 100644 index 0000000000000..9362cc075ea52 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/plugin/PluginBuildPlugin.java @@ -0,0 +1,332 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.plugin; + +import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin; +import groovy.lang.Closure; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.elasticsearch.gradle.BuildPlugin; +import org.elasticsearch.gradle.NoticeTask; +import org.elasticsearch.gradle.Version; +import org.elasticsearch.gradle.VersionProperties; +import org.elasticsearch.gradle.dependencies.CompileOnlyResolvePlugin; +import org.elasticsearch.gradle.info.BuildParams; +import org.elasticsearch.gradle.internal.precommit.TestingConventionsTasks; +import org.elasticsearch.gradle.test.RestTestBasePlugin; +import org.elasticsearch.gradle.testclusters.ElasticsearchCluster; +import org.elasticsearch.gradle.testclusters.RunTask; +import org.elasticsearch.gradle.testclusters.TestClustersPlugin; +import org.elasticsearch.gradle.util.GradleUtils; +import org.elasticsearch.gradle.util.Util; +import org.gradle.api.Action; +import org.gradle.api.GradleException; +import org.gradle.api.InvalidUserDataException; +import org.gradle.api.NamedDomainObjectContainer; +import org.gradle.api.Plugin; +import org.gradle.api.Project; +import org.gradle.api.Task; +import org.gradle.api.Transformer; +import org.gradle.api.file.RegularFile; +import org.gradle.api.plugins.BasePlugin; +import org.gradle.api.plugins.JavaPluginExtension; +import org.gradle.api.provider.Provider; +import org.gradle.api.publish.PublishingExtension; +import org.gradle.api.publish.maven.MavenPublication; +import org.gradle.api.publish.maven.plugins.MavenPublishPlugin; +import org.gradle.api.tasks.Copy; +import org.gradle.api.tasks.SourceSetContainer; +import org.gradle.api.tasks.TaskProvider; +import org.gradle.api.tasks.bundling.Zip; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * Encapsulates build configuration for an Elasticsearch plugin. + */ +public class PluginBuildPlugin implements Plugin { + @Override + public void apply(final Project project) { + project.getPluginManager().apply(BuildPlugin.class); + project.getPluginManager().apply(RestTestBasePlugin.class); + project.getPluginManager().apply(CompileOnlyResolvePlugin.class); + + var extension = project.getExtensions().create(PLUGIN_EXTENSION_NAME, PluginPropertiesExtension.class, project); + configureDependencies(project); + + final var bundleTask = createBundleTasks(project, extension); + + project.afterEvaluate(project1 -> { + project1.getExtensions() + .getByType(PluginPropertiesExtension.class) + .getExtendedPlugins() + .forEach( + pluginName -> { + // Auto add dependent modules to the test cluster + if (project1.findProject(":modules:" + pluginName) != null) { + NamedDomainObjectContainer testClusters = testClusters(project, "testClusters"); + testClusters.all(elasticsearchCluster -> elasticsearchCluster.module(":modules:" + pluginName)); + } + } + ); + final var extension1 = project1.getExtensions().getByType(PluginPropertiesExtension.class); + configurePublishing(project1, extension1); + var name = extension1.getName(); + project1.setProperty("archivesBaseName", name); + project1.setDescription(extension1.getDescription()); + + if (extension1.getName() == null) { + throw new InvalidUserDataException("name is a required setting for esplugin"); + } + + if (extension1.getDescription() == null) { + throw new InvalidUserDataException("description is a required setting for esplugin"); + } + + if (extension1.getType().equals(PluginType.BOOTSTRAP) == false && extension1.getClassname() == null) { + throw new InvalidUserDataException("classname is a required setting for esplugin"); + } + + Map map = new LinkedHashMap<>(12); + map.put("name", extension1.getName()); + map.put("description", extension1.getDescription()); + map.put("version", extension1.getVersion()); + map.put("elasticsearchVersion", Version.fromString(VersionProperties.getElasticsearch()).toString()); + map.put("javaVersion", project1.getExtensions().getByType(JavaPluginExtension.class).getTargetCompatibility().toString()); + map.put("classname", extension1.getType().equals(PluginType.BOOTSTRAP) ? "" : extension1.getClassname()); + map.put("extendedPlugins", extension1.getExtendedPlugins().stream().collect(Collectors.joining(","))); + map.put("hasNativeController", extension1.isHasNativeController()); + map.put("requiresKeystore", extension1.isRequiresKeystore()); + map.put("type", extension1.getType().toString()); + map.put("javaOpts", extension1.getJavaOpts()); + map.put("licensed", extension1.isLicensed()); + project1.getTasks().withType(Copy.class).named("pluginProperties").configure(copy -> { + copy.expand(map); + copy.getInputs().properties(map); + }); + BuildParams.withInternalBuild(() -> { + boolean isModule = GradleUtils.isModuleProject(project1.getPath()); + boolean isXPackModule = isModule && project1.getPath().startsWith(":x-pack"); + if (isModule == false || isXPackModule) { + addNoticeGeneration(project1, extension1); + } + }); + }); + + BuildParams.withInternalBuild(() -> { + // We've ported this from multiple build scripts where we see this pattern into + // an extension method as a first step of consolidation. + // We might want to port this into a general pattern later on. + project.getExtensions() + .getExtraProperties() + .set("addQaCheckDependencies", new Closure(PluginBuildPlugin.this, PluginBuildPlugin.this) { + public void doCall(Object it) { + project.afterEvaluate(project1 -> { + // let check depend on check tasks of qa sub-projects + final var checkTaskProvider = project1.getTasks().named("check"); + Optional qaSubproject = project1.getSubprojects() + .stream() + .filter(p -> p.getPath().equals(project1.getPath() + ":qa")) + .findFirst(); + qaSubproject.ifPresent( + qa -> qa.getSubprojects() + .forEach(p -> checkTaskProvider.configure(task -> task.dependsOn(p.getPath() + ":check"))) + ); + }); + } + + public void doCall() { + doCall(null); + } + }); + + project.getTasks().withType(TestingConventionsTasks.class).named("testingConventions").configure(t -> { + t.getNaming().clear(); + t.getNaming() + .create("Tests", testingConventionRule -> testingConventionRule.baseClass("org.apache.lucene.util.LuceneTestCase")); + t.getNaming().create("IT", testingConventionRule -> { + testingConventionRule.baseClass("org.elasticsearch.test.ESIntegTestCase"); + testingConventionRule.baseClass("org.elasticsearch.test.rest.ESRestTestCase"); + testingConventionRule.baseClass("org.elasticsearch.test.ESSingleNodeTestCase"); + }); + }); + }); + + project.getConfigurations().getByName("default").extendsFrom(project.getConfigurations().getByName("runtimeClasspath")); + + // allow running ES with this plugin in the foreground of a build + var testClusters = testClusters(project, TestClustersPlugin.EXTENSION_NAME); + final var runCluster = testClusters.create("runTask", cluster -> { + if (GradleUtils.isModuleProject(project.getPath())) { + cluster.module(bundleTask.flatMap((Transformer, Zip>) zip -> zip.getArchiveFile())); + } else { + cluster.plugin(bundleTask.flatMap((Transformer, Zip>) zip -> zip.getArchiveFile())); + } + }); + + project.getTasks().register("run", RunTask.class, runTask -> { + runTask.useCluster(runCluster); + runTask.dependsOn(project.getTasks().named("bundlePlugin")); + }); + } + + @SuppressWarnings("unchecked") + private static NamedDomainObjectContainer testClusters(Project project, String extensionName) { + return (NamedDomainObjectContainer) project.getExtensions().getByName(extensionName); + } + + private static void configurePublishing(Project project, PluginPropertiesExtension extension) { + if (project.getPlugins().hasPlugin(MavenPublishPlugin.class)) { + PublishingExtension publishingExtension = project.getExtensions().getByType(PublishingExtension.class); + MavenPublication elastic = publishingExtension.getPublications().maybeCreate("elastic", MavenPublication.class); + elastic.setArtifactId(extension.getName()); + } + } + + private static void configureDependencies(final Project project) { + var dependencies = project.getDependencies(); + if (BuildParams.isInternal()) { + dependencies.add("compileOnly", dependencies.project(Map.of("path", ":server"))); + dependencies.add("testImplementation", dependencies.project(Map.of("path", ":test:framework"))); + } else { + dependencies.add("compileOnly", "org.elasticsearch:elasticsearch:" + VersionProperties.getElasticsearch()); + dependencies.add("testImplementation", ":test:framework:" + VersionProperties.getElasticsearch()); + } + + // we "upgrade" these optional deps to provided for plugins, since they will run + // with a full elasticsearch server that includes optional deps + dependencies.add("compileOnly", "org.locationtech.spatial4j:spatial4j:" + VersionProperties.getVersions().get("spatial4j")); + dependencies.add("compileOnly", "org.locationtech.jts:jts-core:" + VersionProperties.getVersions().get("jts")); + dependencies.add("compileOnly", "org.apache.logging.log4j:log4j-api:" + VersionProperties.getVersions().get("log4j")); + dependencies.add("compileOnly", "org.apache.logging.log4j:log4j-core:" + VersionProperties.getVersions().get("log4j")); + dependencies.add("compileOnly", "org.elasticsearch:jna:" + VersionProperties.getVersions().get("jna")); + } + + /** + * Adds a bundlePlugin task which builds the zip containing the plugin jars, + * metadata, properties, and packaging files + */ + private static TaskProvider createBundleTasks(final Project project, PluginPropertiesExtension extension) { + final var pluginMetadata = project.file("src/main/plugin-metadata"); + final var templateFile = new File(project.getBuildDir(), "templates/plugin-descriptor.properties"); + + // create tasks to build the properties file for this plugin + final var copyPluginPropertiesTemplate = project.getTasks().register("copyPluginPropertiesTemplate", new Action() { + @Override + public void execute(Task task) { + task.getOutputs().file(templateFile); + // intentionally an Action and not a lambda to avoid issues with up-to-date check + task.doLast(new Action() { + @Override + public void execute(Task task) { + InputStream resourceTemplate = PluginBuildPlugin.class.getResourceAsStream("/" + templateFile.getName()); + try { + String templateText = IOUtils.toString(resourceTemplate, StandardCharsets.UTF_8.name()); + FileUtils.write(templateFile, templateText, "UTF-8"); + } catch (IOException e) { + throw new GradleException("Unable to copy plugin properties", e); + } + } + }); + } + }); + final var buildProperties = project.getTasks().register("pluginProperties", Copy.class, copy -> { + copy.dependsOn(copyPluginPropertiesTemplate); + copy.from(templateFile); + copy.into(new File(project.getBuildDir(), "generated-resources")); + }); + // add the plugin properties and metadata to test resources, so unit tests can + // know about the plugin (used by test security code to statically initialize the plugin in unit tests) + var testSourceSet = project.getExtensions().getByType(SourceSetContainer.class).getByName("test"); + Map map = new LinkedHashMap(1); + map.put("builtBy", buildProperties); + testSourceSet.getOutput().dir(map, new File(project.getBuildDir(), "generated-resources")); + testSourceSet.getResources().srcDir(pluginMetadata); + + // create the actual bundle task, which zips up all the files for the plugin + final var bundle = project.getTasks().register("bundlePlugin", Zip.class, zip -> { + zip.from(buildProperties); + zip.from( + pluginMetadata, + copySpec -> { + // metadata (eg custom security policy) + // the codebases properties file is only for tests and not needed in production + copySpec.exclude("plugin-security.codebases"); + } + ); + + /* + * If the plugin is using the shadow plugin then we need to bundle + * that shadow jar. + */ + zip.from(new Closure(null, null) { + public Object doCall(Object it) { + return project.getPlugins().hasPlugin(ShadowPlugin.class) + ? project.getTasks().named("shadowJar") + : project.getTasks().named("jar"); + } + + public Object doCall() { + return doCall(null); + } + + }); + zip.from( + project.getConfigurations() + .getByName("runtimeClasspath") + .minus(project.getConfigurations().getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME)) + ); + // extra files for the plugin to go into the zip + zip.from("src/main/packaging");// TODO: move all config/bin/_size/etc into packaging + zip.from("src/main", copySpec -> { + copySpec.include("config/**"); + copySpec.include("bin/**"); + }); + }); + project.getTasks().named(BasePlugin.ASSEMBLE_TASK_NAME).configure(task -> task.dependsOn(bundle)); + + // also make the zip available as a configuration (used when depending on this project) + project.getConfigurations().create("zip"); + project.getArtifacts().add("zip", bundle); + + return bundle; + } + + /** + * Configure the pom for the main jar of this plugin + */ + protected static void addNoticeGeneration(final Project project, PluginPropertiesExtension extension) { + final var licenseFile = extension.getLicenseFile(); + var tasks = project.getTasks(); + if (licenseFile != null) { + tasks.withType(Zip.class).named("bundlePlugin").configure(zip -> zip.from(licenseFile.getParentFile(), copySpec -> { + copySpec.include(licenseFile.getName()); + copySpec.rename(s -> "LICENSE.txt"); + })); + } + + final var noticeFile = extension.getNoticeFile(); + if (noticeFile != null) { + final var generateNotice = tasks.register("generateNotice", NoticeTask.class, noticeTask -> { + noticeTask.setInputFile(noticeFile); + noticeTask.source(Util.getJavaMainSourceSet(project).get().getAllJava()); + }); + tasks.withType(Zip.class).named("bundlePlugin").configure(task -> task.from(generateNotice)); + } + } + + public static final String PLUGIN_EXTENSION_NAME = "esplugin"; +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/TestWithDependenciesPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/TestWithDependenciesPlugin.java new file mode 100644 index 0000000000000..a3f2deb80d438 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/TestWithDependenciesPlugin.java @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.test; + +import org.apache.commons.lang.StringUtils; +import org.elasticsearch.gradle.plugin.PluginBuildPlugin; +import org.gradle.api.Plugin; +import org.gradle.api.Project; +import org.gradle.api.artifacts.Configuration; +import org.gradle.api.artifacts.ProjectDependency; +import org.gradle.api.plugins.ExtraPropertiesExtension; +import org.gradle.api.tasks.Copy; +import org.gradle.api.tasks.SourceSetContainer; + +import java.io.File; +import java.util.Map; +import java.util.stream.Collectors; + +import static java.util.Arrays.stream; + +/** + * A plugin to run tests that depend on other plugins or modules. + *

+ * This plugin will add the plugin-metadata and properties files for each + * dependency to the test source set. + */ +public class TestWithDependenciesPlugin implements Plugin { + @Override + public void apply(final Project project) { + ExtraPropertiesExtension extraProperties = project.getExtensions().getExtraProperties(); + if (extraProperties.has("isEclipse") && Boolean.valueOf(extraProperties.get("isEclipse").toString())) { + /* The changes this plugin makes both break and aren't needed by + * Eclipse. This is because Eclipse flattens main and test + * dependencies into a single dependency. Because Eclipse is + * "special".... */ + return; + } + + Configuration testImplementationConfig = project.getConfigurations().getByName("testImplementation"); + testImplementationConfig.getDependencies().all(dep -> { + if (dep instanceof ProjectDependency + && ((ProjectDependency) dep).getDependencyProject().getPlugins().hasPlugin(PluginBuildPlugin.class)) { + project.getGradle() + .projectsEvaluated(gradle -> addPluginResources(project, ((ProjectDependency) dep).getDependencyProject())); + } + }); + } + + private static void addPluginResources(final Project project, final Project pluginProject) { + final File outputDir = new File(project.getBuildDir(), "/generated-resources/" + pluginProject.getName()); + String camelProjectName = stream(pluginProject.getName().split("-")).map(t -> StringUtils.capitalize(t)) + .collect(Collectors.joining()); + String taskName = "copy" + camelProjectName + "Metadata"; + project.getTasks().register(taskName, Copy.class, copy -> { + copy.into(outputDir); + copy.from(pluginProject.getTasks().named("pluginProperties")); + copy.from(pluginProject.file("src/main/plugin-metadata")); + }); + + Map map = Map.of("builtBy", taskName); + SourceSetContainer sourceSetContainer = project.getExtensions().getByType(SourceSetContainer.class); + sourceSetContainer.getByName("test").getOutput().dir(map, outputDir); + } +} From 0c6a743db09f0d50264112973d1b8aac133caa31 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Wed, 10 Mar 2021 17:00:11 +0100 Subject: [PATCH 2/9] Fix optional input in NoticeTask --- .../java/org/elasticsearch/gradle/NoticeTask.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/NoticeTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/NoticeTask.java index f54a89769529c..3bc8f622ba9fe 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/NoticeTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/NoticeTask.java @@ -8,7 +8,6 @@ package org.elasticsearch.gradle; -import groovy.lang.Reference; import org.apache.commons.io.FileUtils; import org.codehaus.groovy.runtime.ResourceGroovyMethods; import org.codehaus.groovy.runtime.StringGroovyMethods; @@ -151,15 +150,15 @@ public void generateNotice() throws IOException { @InputFiles @Optional public FileCollection getNoticeFiles() { - final Reference tree = new Reference<>(); - licensesDirs.forEach(dir -> { - if (tree.get() == null) { - tree.set(getProject().fileTree(dir)); + FileTree tree = null; + for (File dir : licensesDirs) { + if (tree == null) { + tree = getProject().fileTree(dir); } else { - tree.set(tree.get().plus(getProject().fileTree(dir))); + tree = tree.plus(getProject().fileTree(dir)); } - }); - return tree.get().matching(patternFilterable -> patternFilterable.include("**/*-NOTICE.txt")); + } + return tree == null ? null : tree.matching(patternFilterable -> patternFilterable.include("**/*-NOTICE.txt")); } @InputFiles From 0756a2fe0910ac44da3e97375ec1c2e4d8d393f9 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Wed, 10 Mar 2021 17:13:09 +0100 Subject: [PATCH 3/9] Next try to fix NoticeTask --- .../org/elasticsearch/gradle/NoticeTask.java | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/NoticeTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/NoticeTask.java index 3bc8f622ba9fe..f3d4448fff7e6 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/NoticeTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/NoticeTask.java @@ -87,16 +87,19 @@ public void generateNotice() throws IOException { // This is a map rather than a set so that the sort order is the 3rd // party component names, unaffected by the full path to the various files final Map seen = new TreeMap(); - for (File file : getNoticeFiles()) { - String name = file.getName().replaceFirst(" -NOTICE\\.txt$ ", ""); - if (seen.containsKey(name)) { - File prevFile = seen.get(name); - String previousFileText = readFileToString(prevFile, "UTF-8"); - if (previousFileText.equals(readFileToString(file, "UTF-8")) == false) { - throw new RuntimeException("Two different notices exist for dependency '" + name + "': " + prevFile + " and " + file); + FileCollection noticeFiles = getNoticeFiles(); + if(noticeFiles != null) { + for (File file : getNoticeFiles()) { + String name = file.getName().replaceFirst(" -NOTICE\\.txt$ ", ""); + if (seen.containsKey(name)) { + File prevFile = seen.get(name); + String previousFileText = readFileToString(prevFile, "UTF-8"); + if (previousFileText.equals(readFileToString(file, "UTF-8")) == false) { + throw new RuntimeException("Two different notices exist for dependency '" + name + "': " + prevFile + " and " + file); + } + } else { + seen.put(name, file); } - } else { - seen.put(name, file); } } From d143b2fd723d52575f9ae2e02da9d9ea858cf60c Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Wed, 10 Mar 2021 22:55:09 +0100 Subject: [PATCH 4/9] Fix regex --- .../src/main/java/org/elasticsearch/gradle/NoticeTask.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/NoticeTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/NoticeTask.java index f3d4448fff7e6..d151bc1ab1737 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/NoticeTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/NoticeTask.java @@ -90,7 +90,8 @@ public void generateNotice() throws IOException { FileCollection noticeFiles = getNoticeFiles(); if(noticeFiles != null) { for (File file : getNoticeFiles()) { - String name = file.getName().replaceFirst(" -NOTICE\\.txt$ ", ""); + String name = file.getName().replaceFirst("-NOTICE\\.txt$", ""); + System.out.println("name = " + name); if (seen.containsKey(name)) { File prevFile = seen.get(name); String previousFileText = readFileToString(prevFile, "UTF-8"); From f8675d78c550e15ee432163d2c6a6eeab4120520 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Wed, 10 Mar 2021 23:15:18 +0100 Subject: [PATCH 5/9] Fix checkstyle --- .../src/main/java/org/elasticsearch/gradle/NoticeTask.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/NoticeTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/NoticeTask.java index d151bc1ab1737..3b7e03532e5df 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/NoticeTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/NoticeTask.java @@ -88,15 +88,16 @@ public void generateNotice() throws IOException { // party component names, unaffected by the full path to the various files final Map seen = new TreeMap(); FileCollection noticeFiles = getNoticeFiles(); - if(noticeFiles != null) { + if (noticeFiles != null) { for (File file : getNoticeFiles()) { String name = file.getName().replaceFirst("-NOTICE\\.txt$", ""); - System.out.println("name = " + name); if (seen.containsKey(name)) { File prevFile = seen.get(name); String previousFileText = readFileToString(prevFile, "UTF-8"); if (previousFileText.equals(readFileToString(file, "UTF-8")) == false) { - throw new RuntimeException("Two different notices exist for dependency '" + name + "': " + prevFile + " and " + file); + throw new RuntimeException( + "Two different notices exist for dependency '" + name + "': " + prevFile + " and " + file + ); } } else { seen.put(name, file); From 74aae2de478b8c33d1d336009abb39da924e1deb Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Thu, 11 Mar 2021 09:36:34 +0100 Subject: [PATCH 6/9] Fix default dependencies for external plugin development --- .../java/org/elasticsearch/gradle/plugin/PluginBuildPlugin.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/plugin/PluginBuildPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/plugin/PluginBuildPlugin.java index 9362cc075ea52..0068b9e9d118f 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/plugin/PluginBuildPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/plugin/PluginBuildPlugin.java @@ -203,7 +203,7 @@ private static void configureDependencies(final Project project) { dependencies.add("testImplementation", dependencies.project(Map.of("path", ":test:framework"))); } else { dependencies.add("compileOnly", "org.elasticsearch:elasticsearch:" + VersionProperties.getElasticsearch()); - dependencies.add("testImplementation", ":test:framework:" + VersionProperties.getElasticsearch()); + dependencies.add("testImplementation", "org.elasticsearch.test:framework:" + VersionProperties.getElasticsearch()); } // we "upgrade" these optional deps to provided for plugins, since they will run From 3978fc825370787720eee96c88ea365a5f19542a Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Thu, 11 Mar 2021 13:55:12 +0100 Subject: [PATCH 7/9] Further simplification in build logic after converting to java Co-authored-by: Rory Hunter --- .../src/main/java/org/elasticsearch/gradle/NoticeTask.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/NoticeTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/NoticeTask.java index 3b7e03532e5df..44e55048a831d 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/NoticeTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/NoticeTask.java @@ -106,12 +106,10 @@ public void generateNotice() throws IOException { } // Add all LICENSE and NOTICE files in licenses directory - for (Map.Entry entry : seen.entrySet()) { - final String name = entry.getKey(); - File file = entry.getValue(); + seen.forEach((name, file) -> { appendFile(file, name, "NOTICE", output); appendFile(new File(file.getParentFile(), name + "-LICENSE.txt"), name, "LICENSE", output); - } + }); // Find any source files with "@notice" annotated license header for (File sourceFile : sources.getFiles()) { From 5fbda82c29043889ccbd327c75dbc055f719e5a1 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Thu, 11 Mar 2021 13:55:37 +0100 Subject: [PATCH 8/9] More simplification Co-authored-by: Rory Hunter --- .../org/elasticsearch/gradle/plugin/PluginBuildPlugin.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/plugin/PluginBuildPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/plugin/PluginBuildPlugin.java index 0068b9e9d118f..140069a3dba6f 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/plugin/PluginBuildPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/plugin/PluginBuildPlugin.java @@ -251,8 +251,7 @@ public void execute(Task task) { // add the plugin properties and metadata to test resources, so unit tests can // know about the plugin (used by test security code to statically initialize the plugin in unit tests) var testSourceSet = project.getExtensions().getByType(SourceSetContainer.class).getByName("test"); - Map map = new LinkedHashMap(1); - map.put("builtBy", buildProperties); + Map map = Map.of("builtBy", buildProperties); testSourceSet.getOutput().dir(map, new File(project.getBuildDir(), "generated-resources")); testSourceSet.getResources().srcDir(pluginMetadata); From bd922190532041c1e675bb8b5de0ecc3daee727d Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Thu, 11 Mar 2021 15:40:40 +0100 Subject: [PATCH 9/9] Centralize FileUtil functions --- .../org/elasticsearch/gradle/NoticeTask.java | 9 +++---- .../elasticsearch/gradle/util/FileUtils.java | 25 +++++++++++++++++++ 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/NoticeTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/NoticeTask.java index 44e55048a831d..9308321335e5c 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/NoticeTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/NoticeTask.java @@ -8,8 +8,6 @@ package org.elasticsearch.gradle; -import org.apache.commons.io.FileUtils; -import org.codehaus.groovy.runtime.ResourceGroovyMethods; import org.codehaus.groovy.runtime.StringGroovyMethods; import org.gradle.api.DefaultTask; import org.gradle.api.file.FileCollection; @@ -21,6 +19,7 @@ import org.gradle.api.tasks.OutputFile; import org.gradle.api.tasks.TaskAction; +import org.elasticsearch.gradle.util.FileUtils; import java.io.File; import java.io.IOException; import java.util.ArrayList; @@ -119,7 +118,7 @@ public void generateNotice() throws IOException { StringBuilder header = new StringBuilder(); String packageDeclaration = null; - for (String line : FileUtils.readLines(sourceFile)) { + for (String line : FileUtils.readLines(sourceFile, "UTF-8")) { if (isPackageInfo && packageDeclaration == null && line.startsWith("package")) { packageDeclaration = line; } @@ -170,8 +169,8 @@ public FileCollection getSources() { return sources; } - public static void appendFile(File file, String name, String type, StringBuilder output) throws IOException { - String text = ResourceGroovyMethods.getText(file, "UTF-8"); + public static void appendFile(File file, String name, String type, StringBuilder output) { + String text = FileUtils.read(file, "UTF-8"); if (text.trim().isEmpty()) { return; } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/util/FileUtils.java b/buildSrc/src/main/java/org/elasticsearch/gradle/util/FileUtils.java index 48df95979a95b..534d28c02c08d 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/util/FileUtils.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/util/FileUtils.java @@ -11,6 +11,7 @@ import org.gradle.api.UncheckedIOException; import java.io.File; +import java.io.IOException; import java.util.Collections; import java.util.LinkedList; import java.util.List; @@ -64,4 +65,28 @@ public static void mkdirs(File dir) { throw new UncheckedIOException(String.format("Failed to create directory '%s'", dir)); } } + + public static String read(File file, String encoding) { + try { + return org.apache.commons.io.FileUtils.readFileToString(file, encoding); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + public static List readLines(File file, String encoding) { + try { + return org.apache.commons.io.FileUtils.readLines(file, encoding); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + public static void write(File outputFile, CharSequence content, String encoding) { + try { + org.apache.commons.io.FileUtils.write(outputFile, content, encoding); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } }