Increase flexibility of test cluster execution environments (#99437)

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalTestArtifactPlugin.java

@@ -23,7 +23,10 @@ public class InternalTestArtifactPlugin implements Plugin<Project> {
     public void apply(Project project) {
         project.getPlugins().apply(InternalTestArtifactBasePlugin.class);
         InternalTestArtifactExtension testArtifactExtension = project.getExtensions().getByType(InternalTestArtifactExtension.class);
-        SourceSet testSourceSet = project.getExtensions().getByType(SourceSetContainer.class).getByName("test");
-        testArtifactExtension.registerTestArtifactFromSourceSet(testSourceSet);
+        project.getExtensions().getByType(SourceSetContainer.class).all(sourceSet -> {
+            if (sourceSet.getName().equals(SourceSet.MAIN_SOURCE_SET_NAME) == false) {
+                testArtifactExtension.registerTestArtifactFromSourceSet(sourceSet);
+            }
+        });
     }
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java

@@ -37,6 +37,7 @@
 import org.gradle.api.artifacts.Dependency;
 import org.gradle.api.artifacts.ProjectDependency;
 import org.gradle.api.artifacts.type.ArtifactTypeDefinition;
+import org.gradle.api.attributes.Attribute;
 import org.gradle.api.file.FileTree;
 import org.gradle.api.provider.ProviderFactory;
 import org.gradle.api.tasks.ClasspathNormalizer;
@@ -69,6 +70,7 @@ public class RestTestBasePlugin implements Plugin<Project> {
     private static final String MODULES_CONFIGURATION = "clusterModules";
     private static final String PLUGINS_CONFIGURATION = "clusterPlugins";
     private static final String EXTRACTED_PLUGINS_CONFIGURATION = "extractedPlugins";
+    private static final Attribute<String> CONFIGURATION_ATTRIBUTE = Attribute.of("test-cluster-artifacts", String.class);
 
     private final ProviderFactory providerFactory;
 
@@ -249,6 +251,7 @@ private Optional<String> findModulePath(Project project, String pluginName) {
 
     private Configuration createPluginConfiguration(Project project, String name, boolean useExploded, boolean isExtended) {
         return project.getConfigurations().create(name, c -> {
+            c.attributes(a -> a.attribute(CONFIGURATION_ATTRIBUTE, name));
             if (useExploded) {
                 c.attributes(a -> a.attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.DIRECTORY_TYPE));
             } else {

modules/build.gradle

@@ -10,6 +10,7 @@ configure(subprojects.findAll { it.parent.path == project.path }) {
   group = 'org.elasticsearch.plugin' // for modules which publish client jars
   apply plugin: 'elasticsearch.internal-testclusters'
   apply plugin: 'elasticsearch.internal-es-plugin'
+  apply plugin: 'elasticsearch.internal-test-artifact'
 
   esplugin {
     // for local ES plugins, the name of the plugin is the same as the directory

qa/full-cluster-restart/build.gradle

@@ -10,13 +10,9 @@ import org.elasticsearch.gradle.internal.info.BuildParams
 import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
 
 apply plugin: 'elasticsearch.internal-java-rest-test'
-apply plugin: 'elasticsearch.internal-test-artifact-base'
+apply plugin: 'elasticsearch.internal-test-artifact'
 apply plugin: 'elasticsearch.bwc-test'
 
-testArtifacts {
-  registerTestArtifactFromSourceSet(sourceSets.javaRestTest)
-}
-
 BuildParams.bwcVersions.withIndexCompatible { bwcVersion, baseName ->
   tasks.register(bwcTaskName(bwcVersion), StandaloneRestIntegTestTask) {
     usesBwcDistribution(bwcVersion)

rest-api-spec/build.gradle

@@ -1,11 +1,10 @@
-import org.elasticsearch.gradle.Version
-
 apply plugin: 'elasticsearch.build'
 apply plugin: 'elasticsearch.publish'
 apply plugin: 'elasticsearch.rest-resources'
 apply plugin: 'elasticsearch.validate-rest-spec'
 apply plugin: 'elasticsearch.internal-yaml-rest-test'
 apply plugin: 'elasticsearch.yaml-rest-compat-test'
+apply plugin: 'elasticsearch.internal-test-artifact'
 
 restResources {
   restTests {

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -1294,6 +1294,15 @@ protected Settings restClientSettings() {
         if (System.getProperty("tests.rest.client_path_prefix") != null) {
             builder.put(CLIENT_PATH_PREFIX, System.getProperty("tests.rest.client_path_prefix"));
         }
+        if (System.getProperty("tests.rest.cluster.username") != null) {
+            if (System.getProperty("tests.rest.cluster.password") == null) {
+                throw new IllegalStateException("The 'test.rest.cluster.password' system property must be set.");
+            }
+            String username = System.getProperty("tests.rest.cluster.username");
+            String password = System.getProperty("tests.rest.cluster.password");
+            String token = basicAuthHeaderValue(username, new SecureString(password.toCharArray()));
+            return builder.put(ThreadContext.PREFIX + ".Authorization", token).build();
+        }
         return builder.build();
     }
 

test/test-clusters/src/main/java/org/elasticsearch/test/cluster/ClusterHandle.java

@@ -8,10 +8,7 @@
 
 package org.elasticsearch.test.cluster;
 
-import org.elasticsearch.test.cluster.util.Version;
-
 import java.io.Closeable;
-import java.io.InputStream;
 
 /**
  * A handle to an {@link ElasticsearchCluster}.
@@ -30,19 +27,6 @@ public interface ClusterHandle extends Closeable {
      */
     void stop(boolean forcibly);
 
-    /**
-     * Stops the node at a given index.
-     * @param index of the node to stop
-     */
-    void stopNode(int index, boolean forcibly);
-
-    /**
-     * Restarts the cluster. Effectively the same as calling {@link #stop(boolean)} followed by {@link #start()}
-     *
-     * @param forcibly whether to ficibly terminate the cluster
-     */
-    void restart(boolean forcibly);
-
     /**
      * Whether the cluster is started or not. This method makes no guarantees on cluster availability, only that the node processes have
      * been started.
@@ -67,74 +51,9 @@ public interface ClusterHandle extends Closeable {
      */
     String getHttpAddress(int index);
 
-    /**
-     * Get the name of the node for the given index.
-     */
-    String getName(int index);
-
-    /**
-     * Get the pid of the node for the given index.
-     */
-    long getPid(int index);
-
-    /**
-     * Returns a comma-separated list of TCP transport endpoints for cluster. If this method is called on an unstarted cluster, the cluster
-     * will be started. This method is thread-safe and subsequent calls will wait for cluster start and availability.\
-     *
-     * @return cluster node TCP transport endpoints
-     */
-    String getTransportEndpoints();
-
-    /**
-     * Returns the TCP transport endpoint for the node at the given index. If this method is called on an unstarted cluster, the cluster
-     * will be started. This method is thread-safe and subsequent calls will wait for cluster start and availability.
-     *
-     * @return cluster node TCP transport endpoints
-     */
-    String getTransportEndpoint(int index);
-
-    /**
-     * Returns a comma-separated list of remote cluster server endpoints for cluster. If this method is called on an unstarted cluster,
-     * the cluster will be started. This method is thread-safe and subsequent calls will wait for cluster start and availability.
-     * Note individual node can enable or disable remote cluster server independently. When a node has remote cluster server disabled,
-     * an empty string is returned for that node. Hence, it is possible for this method to return something like "[::1]:63300,,".
-     *
-     * @return cluster node remote cluster server endpoints
-     */
-    String getRemoteClusterServerEndpoints();
-
-    /**
-     * Returns the remote cluster server endpoint for the node at the given index. If this method is called on an unstarted cluster,
-     * the cluster will be started. This method is thread-safe and subsequent calls will wait for cluster start and availability.
-     * Note individual node can enable or disable remote cluster server independently. When a node has remote cluster server disabled,
-     * an empty string is returned.
-     *
-     * @return cluster node remote cluster server endpoints
-     */
-    String getRemoteClusterServerEndpoint(int index);
-
-    /**
-     * Upgrades a single node to the given version. Method blocks until the node is back up and ready to respond to requests.
-     *
-     * @param index index of node ot upgrade
-     * @param version version to upgrade to
-     */
-    void upgradeNodeToVersion(int index, Version version);
-
-    /**
-     * Performs a "full cluster restart" upgrade to the given version. Method blocks until the cluster is restarted and available.
-     *
-     * @param version version to upgrade to
-     */
-    void upgradeToVersion(Version version);
-
     /**
      * Cleans up any resources created by this cluster. Calling this method will forcibly terminate any running nodes.
      */
     void close();
 
-    /**
-     * Returns an {@link InputStream} for the given node log.
-     */
-    InputStream getNodeLog(int index, LogType logType);
 }

test/test-clusters/src/main/java/org/elasticsearch/test/cluster/ElasticsearchCluster.java

@@ -9,9 +9,14 @@
 package org.elasticsearch.test.cluster;
 
 import org.elasticsearch.test.cluster.local.DefaultLocalClusterSpecBuilder;
+import org.elasticsearch.test.cluster.local.LocalClusterHandle;
 import org.elasticsearch.test.cluster.local.LocalClusterSpecBuilder;
 import org.junit.rules.TestRule;
 
+import java.util.List;
+import java.util.ServiceLoader;
+import java.util.stream.Collectors;
+
 /**
  * <p>A JUnit test rule for orchestrating an Elasticsearch cluster for local integration testing. New clusters can be created via one of the
  * various static builder methods. For example:</p>
@@ -20,7 +25,7 @@
  * public static ElasticsearchCluster myCluster = ElasticsearchCluster.local().build();
  * </pre>
  */
-public interface ElasticsearchCluster extends TestRule, ClusterHandle {
+public interface ElasticsearchCluster extends TestRule, LocalClusterHandle {
 
     /**
      * Creates a new {@link LocalClusterSpecBuilder} for defining a locally orchestrated cluster. Local clusters use a locally built
@@ -29,7 +34,21 @@ public interface ElasticsearchCluster extends TestRule, ClusterHandle {
      * @return a builder for a local cluster
      */
     static LocalClusterSpecBuilder<ElasticsearchCluster> local() {
-        return new DefaultLocalClusterSpecBuilder();
+        return locateBuilderImpl();
     }
 
+    @SuppressWarnings({ "unchecked", "rawtypes" })
+    private static LocalClusterSpecBuilder<ElasticsearchCluster> locateBuilderImpl() {
+        ServiceLoader<LocalClusterSpecBuilder> loader = ServiceLoader.load(LocalClusterSpecBuilder.class);
+        List<ServiceLoader.Provider<LocalClusterSpecBuilder>> providers = loader.stream().toList();
+
+        if (providers.isEmpty()) {
+            return new DefaultLocalClusterSpecBuilder();
+        } else if (providers.size() > 1) {
+            String providerTypes = providers.stream().map(p -> p.type().getName()).collect(Collectors.joining(","));
+            throw new IllegalStateException("Located multiple LocalClusterSpecBuilder providers [" + providerTypes + "]");
+        }
+
+        return providers.get(0).get();
+    }
 }

test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java

@@ -12,7 +12,6 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.test.cluster.ClusterFactory;
 import org.elasticsearch.test.cluster.LogType;
 import org.elasticsearch.test.cluster.local.LocalClusterSpec.LocalNodeSpec;
 import org.elasticsearch.test.cluster.local.distribution.DistributionDescriptor;
@@ -63,7 +62,7 @@
 
 public abstract class AbstractLocalClusterFactory<S extends LocalClusterSpec, H extends LocalClusterHandle>
     implements
-        ClusterFactory<S, H> {
+        LocalClusterFactory<S, H> {
     private static final Logger LOGGER = LogManager.getLogger(AbstractLocalClusterFactory.class);
     private static final Duration NODE_UP_TIMEOUT = Duration.ofMinutes(2);
     private static final Map<Pair<Version, DistributionType>, DistributionDescriptor> TEST_DISTRIBUTIONS = new ConcurrentHashMap<>();
@@ -106,17 +105,25 @@ public static class Node {
         private final Path logsDir;
         private final Path configDir;
         private final Path tempDir;
+        private final boolean usesSecureSecretsFile;
 
         private Path distributionDir;
         private Version currentVersion;
         private Process process = null;
         private DistributionDescriptor distributionDescriptor;
 
         public Node(Path baseWorkingDir, DistributionResolver distributionResolver, LocalNodeSpec spec) {
-            this(baseWorkingDir, distributionResolver, spec, null);
+            this(baseWorkingDir, distributionResolver, spec, null, false);
         }
 
-        public Node(Path baseWorkingDir, DistributionResolver distributionResolver, LocalNodeSpec spec, String suffix) {
+        public Node(
+            Path baseWorkingDir,
+            DistributionResolver distributionResolver,
+            LocalNodeSpec spec,
+            String suffix,
+            boolean usesSecureSecretsFile
+        ) {
+            this.usesSecureSecretsFile = usesSecureSecretsFile;
             this.objectMapper = new ObjectMapper();
             this.baseWorkingDir = baseWorkingDir;
             this.distributionResolver = distributionResolver;
@@ -155,10 +162,13 @@ public synchronized void start(Version version) {
             }
 
             writeConfiguration();
-            createKeystore();
-            addKeystoreSettings();
-            addKeystoreFiles();
-            writeSecureSecretsFile();
+            if (usesSecureSecretsFile) {
+                writeSecureSecretsFile();
+            } else {
+                createKeystore();
+                addKeystoreSettings();
+                addKeystoreFiles();
+            }
             configureSecurity();
 
             startElasticsearch();
@@ -476,12 +486,20 @@ private void addKeystoreFiles() {
         }
 
         private void writeSecureSecretsFile() {
-            if (spec.getSecrets().isEmpty() == false) {
+            if (spec.getKeystoreFiles().isEmpty() == false) {
+                throw new IllegalStateException(
+                    "Non-string secure secrets are not supported in serverless. Secrets: ["
+                        + spec.getKeystoreFiles().keySet().stream().collect(Collectors.joining(","))
+                        + "]"
+                );
+            }
+            Map<String, String> secrets = spec.resolveKeystore();
+            if (secrets.isEmpty() == false) {
                 try {
                     Path secretsFile = configDir.resolve("secrets/secrets.json");
                     Files.createDirectories(secretsFile.getParent());
                     Map<String, Object> secretsFileContent = new HashMap<>();
-                    secretsFileContent.put("secrets", spec.getSecrets());
+                    secretsFileContent.put("secrets", secrets);
                     secretsFileContent.put("metadata", Map.of("version", "1", "compatibility", spec.getVersion().toString()));
                     Files.writeString(secretsFile, objectMapper.writeValueAsString(secretsFileContent));
                 } catch (IOException e) {

test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterSpecBuilder.java

@@ -161,8 +161,7 @@ private LocalNodeSpec build(LocalClusterSpec cluster) {
                 getKeystorePassword(),
                 getExtraConfigFiles(),
                 getSystemProperties(),
-                getJvmArgs(),
-                getSecrets()
+                getJvmArgs()
             );
         }
     }

test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalSpecBuilder.java

@@ -39,7 +39,6 @@ public abstract class AbstractLocalSpecBuilder<T extends LocalSpecBuilder<?>> im
     private final Map<String, Resource> extraConfigFiles = new HashMap<>();
     private final Map<String, String> systemProperties = new HashMap<>();
     private final List<String> jvmArgs = new ArrayList<>();
-    private final Map<String, String> secrets = new HashMap<>();
     private DistributionType distributionType;
     private Version version;
     private String keystorePassword;
@@ -189,16 +188,6 @@ public List<SettingsProvider> getKeystoreProviders() {
         return inherit(() -> parent.getKeystoreProviders(), keystoreProviders);
     }
 
-    @Override
-    public T secret(String key, String value) {
-        this.secrets.put(key, value);
-        return cast(this);
-    }
-
-    public Map<String, String> getSecrets() {
-        return inherit(() -> parent.getSecrets(), secrets);
-    }
-
     @Override
     public T configFile(String fileName, Resource configFile) {
         this.extraConfigFiles.put(fileName, configFile);

test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalClusterFactory.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test.cluster.local;
+
+import org.elasticsearch.test.cluster.local.distribution.DistributionResolver;
+
+import java.nio.file.Path;
+
+public class DefaultLocalClusterFactory extends AbstractLocalClusterFactory<LocalClusterSpec, DefaultLocalClusterHandle> {
+    private final DistributionResolver distributionResolver;
+
+    public DefaultLocalClusterFactory(DistributionResolver distributionResolver) {
+        super(distributionResolver);
+        this.distributionResolver = distributionResolver;
+    }
+
+    protected DefaultLocalClusterHandle createHandle(Path baseWorkingDir, LocalClusterSpec spec) {
+        return new DefaultLocalClusterHandle(
+            spec.getName(),
+            spec.getNodes().stream().map(s -> new Node(baseWorkingDir, distributionResolver, s)).toList()
+        );
+    }
+}