Use explodedBundle for test cluster module configurations (#84997)

This removes the overhead of zipping modules that are referenced in
test cluster configurations
We now Support declaring task providers as module input in testclusters

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/InternalYamlRestTestPluginFuncTest.groovy

@@ -8,6 +8,7 @@
 
 package org.elasticsearch.gradle.internal.test.rest
 
+import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.fixtures.AbstractRestResourcesFuncTest
 import org.gradle.testkit.runner.TaskOutcome
 
@@ -78,4 +79,121 @@ class InternalYamlRestTestPluginFuncTest extends AbstractRestResourcesFuncTest {
         result.task(':copyRestApiSpecsTask').outcome == TaskOutcome.UP_TO_DATE
         result.task(':copyYamlTestsTask').outcome == TaskOutcome.NO_SOURCE
     }
+
+    def "#type projects are wired into test cluster setup"() {
+        given:
+        internalBuild()
+        localDistroSetup()
+        def distroVersion = VersionProperties.getElasticsearch()
+
+        def subProjectBuildFile = addSubProject(pluginProjectPath)
+        subProjectBuildFile << """
+            apply plugin: 'elasticsearch.esplugin'
+            apply plugin: 'elasticsearch.internal-yaml-rest-test'
+
+            dependencies {
+               yamlRestTestImplementation "junit:junit:4.12"
+            }
+           
+            esplugin { 
+                description = 'test plugin'
+                classname = 'com.acme.plugin.TestPlugin'
+            }
+            
+            // for testing purposes only
+            configurations.compileOnly.dependencies.clear()
+            
+            testClusters {
+              yamlRestTest {
+                  version = "$distroVersion"
+                  testDistribution = 'INTEG_TEST'
+              }
+            }
+        """
+        def testFile = new File(subProjectBuildFile.parentFile, 'src/yamlRestTest/java/org/acme/SomeTestIT.java')
+        testFile.parentFile.mkdirs()
+        testFile << """
+        package org.acme;
+        
+        import org.junit.Test;
+        
+        public class SomeTestIT {
+            @Test
+            public void someMethod() {
+            }
+        }
+        """
+
+        when:
+        def result = gradleRunner("yamlRestTest", "--console", 'plain', '--stacktrace').buildAndFail()
+
+        then:
+        result.task(":distribution:archives:integ-test-zip:buildExpanded").outcome == TaskOutcome.SUCCESS
+        result.getOutput().contains(expectedInstallLog)
+
+        where:
+        type     | pluginProjectPath   | expectedInstallLog
+        "plugin" | ":plugins:plugin-a" | "installing 1 plugins in a single transaction"
+        "module" | ":modules:module-a" | "Installing 1 modules"
+    }
+
+    private void localDistroSetup() {
+        settingsFile << """
+        include ":distribution:archives:integ-test-zip"
+        """
+        def distProjectFolder = file("distribution/archives/integ-test-zip")
+        file(distProjectFolder, 'current-marker.txt') << "current"
+
+        def elasticPluginScript = file(distProjectFolder, 'src/bin/elasticsearch-plugin')
+        elasticPluginScript << """#!/bin/bash
+@echo off
+echo "Installing plugin \$0"
+"""
+        assert elasticPluginScript.setExecutable(true)
+
+        def elasticKeystoreScript = file(distProjectFolder, 'src/bin/elasticsearch-keystore')
+        elasticKeystoreScript << """#!/bin/bash
+@echo off
+echo "Installing keystore \$0"
+"""
+        assert elasticKeystoreScript.setExecutable(true)
+
+        def elasticScript = file(distProjectFolder, 'src/bin/elasticsearch')
+        elasticScript << """#!/bin/bash
+@echo off
+echo "Running elasticsearch \$0"
+"""
+        assert elasticScript.setExecutable(true)
+
+        file(distProjectFolder, 'src/config/elasticsearch.properties') << "some propes"
+        file(distProjectFolder, 'src/config/jvm.options') << """
+-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m
+-XX:ErrorFile=logs/hs_err_pid%p.log
+-XX:HeapDumpPath=data
+"""
+        file(distProjectFolder, 'build.gradle') << """
+            import org.gradle.api.internal.artifacts.ArtifactAttributes;
+
+            apply plugin:'distribution'
+            def buildExpanded = tasks.register("buildExpanded", Copy) {
+                into("build/local")
+                
+                into('es-dummy-dist') {
+                    from('src')
+                    from('current-marker.txt')
+                }
+            }
+            
+            configurations {
+                extracted {
+                    attributes {
+                          attribute(ArtifactAttributes.ARTIFACT_FORMAT, "directory")
+                    }
+                }
+            }
+            artifacts {
+                it.add("extracted", buildExpanded)
+            }
+        """
+    }
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/RestTestBasePlugin.java

@@ -22,12 +22,14 @@
 import org.gradle.api.Project;
 import org.gradle.api.plugins.JavaBasePlugin;
 import org.gradle.api.provider.ProviderFactory;
-import org.gradle.api.tasks.TaskProvider;
-import org.gradle.api.tasks.bundling.AbstractArchiveTask;
+import org.gradle.api.tasks.Sync;
 import org.gradle.api.tasks.bundling.Zip;
 
 import javax.inject.Inject;
 
+import static org.elasticsearch.gradle.plugin.PluginBuildPlugin.BUNDLE_PLUGIN_TASK_NAME;
+import static org.elasticsearch.gradle.plugin.PluginBuildPlugin.EXPLODED_BUNDLE_PLUGIN_TASK_NAME;
+
 public class RestTestBasePlugin implements Plugin<Project> {
     private static final String TESTS_REST_CLUSTER = "tests.rest.cluster";
     private static final String TESTS_CLUSTER = "tests.cluster";
@@ -83,14 +85,13 @@ public void apply(Project project) {
         project.getTasks().withType(StandaloneRestIntegTestTask.class).configureEach(t ->
         // if this a module or plugin, it may have an associated zip file with it's contents, add that to the test cluster
         project.getPluginManager().withPlugin("elasticsearch.esplugin", plugin -> {
-            TaskProvider<Zip> bundle = project.getTasks().withType(Zip.class).named("bundlePlugin");
-            t.dependsOn(bundle);
             if (GradleUtils.isModuleProject(project.getPath())) {
-                t.getClusters().forEach(c -> c.module(bundle.flatMap(AbstractArchiveTask::getArchiveFile)));
+                var bundle = project.getTasks().withType(Sync.class).named(EXPLODED_BUNDLE_PLUGIN_TASK_NAME);
+                t.getClusters().forEach(c -> c.module(bundle));
             } else {
-                t.getClusters().forEach(c -> c.plugin(bundle.flatMap(AbstractArchiveTask::getArchiveFile)));
+                var bundle = project.getTasks().withType(Zip.class).named(BUNDLE_PLUGIN_TASK_NAME);
+                t.getClusters().forEach(c -> c.plugin(bundle));
             }
-
         }));
     }
 

build-tools/src/integTest/groovy/org/elasticsearch/gradle/TestClustersPluginFuncTest.groovy

@@ -120,9 +120,9 @@ class TestClustersPluginFuncTest extends AbstractGradleFuncTest {
     }
 
     @Unroll
-    def "test cluster modules #propertyName change is detected"() {
+    def "test cluster #pluginType #propertyName change is detected"() {
         given:
-        addSubProject("test-module") << """
+        addSubProject("test-$pluginType") << """
             plugins {
                 id 'elasticsearch.esplugin'
             }
@@ -131,9 +131,9 @@ class TestClustersPluginFuncTest extends AbstractGradleFuncTest {
             configurations.testImplementation.dependencies.clear()
             
             esplugin {
-                name = 'test-module'
+                name = 'test-$pluginType'
                 classname 'org.acme.TestModule'
-                description = "test module description"
+                description = "test $pluginType description"
             }
             
             version = "1.0"
@@ -143,7 +143,7 @@ class TestClustersPluginFuncTest extends AbstractGradleFuncTest {
             testClusters {
               myCluster {
                 testDistribution = 'default'
-                module ':test-module'
+                $pluginType ':test-$pluginType'
               }
             }
 
@@ -170,9 +170,11 @@ class TestClustersPluginFuncTest extends AbstractGradleFuncTest {
         assertEsLogContains("myCluster", "Stopping node")
 
         where:
-        propertyName         | fileChange
-        "installedFiles"     | { def testClazz -> testClazz.file("test-module/src/main/plugin-metadata/someAddedConfig.txt") << "new resource file" }
-        "installedClasspath" | { def testClazz -> testClazz.file("test-module/src/main/java/SomeClass.java") << "class SomeClass {}" }
+        pluginType | propertyName         | fileChange
+        'module'   | "installedFiles"     | { def testClazz -> testClazz.file("test-module/src/main/plugin-metadata/someAddedConfig.txt") << "new resource file" }
+        'plugin'   | "installedFiles"     | { def testClazz -> testClazz.file("test-plugin/src/main/plugin-metadata/someAddedConfig.txt") << "new resource file" }
+        'module'   | "installedClasspath" | { def testClazz -> testClazz.file("test-module/src/main/java/SomeClass.java") << "class SomeClass {}" }
+        'plugin'   | "installedClasspath" | { def testClazz -> testClazz.file("test-plugin/src/main/java/SomeClass.java") << "class SomeClass {}" }
     }
 
     def "can declare test cluster in lazy evaluated task configuration block"() {

build-tools/src/main/java/org/elasticsearch/gradle/plugin/PluginBuildPlugin.java

@@ -59,6 +59,7 @@ public class PluginBuildPlugin implements Plugin<Project> {
 
     public static final String PLUGIN_EXTENSION_NAME = "esplugin";
     public static final String BUNDLE_PLUGIN_TASK_NAME = "bundlePlugin";
+    public static final String EXPLODED_BUNDLE_PLUGIN_TASK_NAME = "explodedBundlePlugin";
     public static final String EXPLODED_BUNDLE_CONFIG = "explodedBundleZip";
 
     @Override
@@ -207,9 +208,9 @@ public void execute(Task task) {
         configuration.getAttributes().attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.ZIP_TYPE);
         project.getArtifacts().add("zip", bundle);
 
-        var explodedBundle = project.getTasks().register("explodedBundlePlugin", Sync.class, sync -> {
+        var explodedBundle = project.getTasks().register(EXPLODED_BUNDLE_PLUGIN_TASK_NAME, Sync.class, sync -> {
             sync.with(bundleSpec);
-            sync.into(new File(project.getBuildDir(), "explodedBundle"));
+            sync.into(new File(project.getBuildDir(), "explodedBundle/" + extension.getName()));
         });
 
         // also make the exploded bundle available as a configuration (used when depending on this project)

build-tools/src/main/java/org/elasticsearch/gradle/test/YamlRestTestPlugin.java

@@ -14,6 +14,7 @@
 import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask;
 import org.elasticsearch.gradle.testclusters.TestClustersPlugin;
 import org.elasticsearch.gradle.transform.UnzipTransform;
+import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.NamedDomainObjectContainer;
 import org.gradle.api.NamedDomainObjectProvider;
 import org.gradle.api.Plugin;
@@ -28,13 +29,15 @@
 import org.gradle.api.tasks.Copy;
 import org.gradle.api.tasks.SourceSet;
 import org.gradle.api.tasks.SourceSetContainer;
+import org.gradle.api.tasks.Sync;
 import org.gradle.api.tasks.TaskProvider;
 import org.gradle.api.tasks.bundling.Zip;
 import org.gradle.language.base.plugins.LifecycleBasePlugin;
 
 import java.io.File;
 
 import static org.elasticsearch.gradle.plugin.PluginBuildPlugin.BUNDLE_PLUGIN_TASK_NAME;
+import static org.elasticsearch.gradle.plugin.PluginBuildPlugin.EXPLODED_BUNDLE_PLUGIN_TASK_NAME;
 
 public class YamlRestTestPlugin implements Plugin<Project> {
 
@@ -81,9 +84,13 @@ public void apply(Project project) {
         var cluster = testClusters.register(YAML_REST_TEST);
         TaskProvider<StandaloneRestIntegTestTask> yamlRestTestTask = setupTestTask(project, testSourceSet, cluster);
         project.getPlugins().withType(PluginBuildPlugin.class, p -> {
-            TaskProvider<Zip> bundle = project.getTasks().withType(Zip.class).named(BUNDLE_PLUGIN_TASK_NAME);
-            cluster.configure(c -> c.plugin(bundle.flatMap(Zip::getArchiveFile)));
-            yamlRestTestTask.configure(t -> t.dependsOn(bundle));
+            if (GradleUtils.isModuleProject(project.getPath())) {
+                var bundle = project.getTasks().withType(Sync.class).named(EXPLODED_BUNDLE_PLUGIN_TASK_NAME);
+                cluster.configure(c -> c.module(bundle));
+            } else {
+                var bundle = project.getTasks().withType(Zip.class).named(BUNDLE_PLUGIN_TASK_NAME);
+                cluster.configure(c -> c.plugin(bundle));
+            }
         });
 
         // Wire up to check task

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java

@@ -24,6 +24,9 @@
 import org.gradle.api.provider.Provider;
 import org.gradle.api.tasks.Internal;
 import org.gradle.api.tasks.Nested;
+import org.gradle.api.tasks.Sync;
+import org.gradle.api.tasks.TaskProvider;
+import org.gradle.api.tasks.bundling.Zip;
 import org.gradle.process.ExecOperations;
 
 import java.io.File;
@@ -189,6 +192,11 @@ public void plugin(Provider<RegularFile> plugin) {
         nodes.all(each -> each.plugin(plugin));
     }
 
+    @Override
+    public void plugin(TaskProvider<Zip> plugin) {
+        nodes.all(each -> each.plugin(plugin));
+    }
+
     @Override
     public void plugin(String pluginProjectPath) {
         nodes.all(each -> each.plugin(pluginProjectPath));
@@ -199,6 +207,11 @@ public void module(Provider<RegularFile> module) {
         nodes.all(each -> each.module(module));
     }
 
+    @Override
+    public void module(TaskProvider<Sync> module) {
+        nodes.all(each -> each.module(module));
+    }
+
     @Override
     public void module(String moduleProjectPath) {
         nodes.all(each -> each.module(moduleProjectPath));