Allow test clusters to run with TLS

CHANGELOG.md

@@ -130,6 +130,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Adds support for tracing runnable scenarios ([#8831](https://github.com/opensearch-project/OpenSearch/pull/8831))
 - [Refactor] MediaTypeParser to MediaTypeParserRegistry ([#8636](https://github.com/opensearch-project/OpenSearch/pull/8636))
 - Create separate SourceLookup instance per segment slice in SignificantTextAggregatorFactory ([#8807](https://github.com/opensearch-project/OpenSearch/pull/8807))
+- Allow test clusters to run with TLS ([#8900](https://github.com/opensearch-project/OpenSearch/pull/8900))
 - Add support for aggregation profiler with concurrent aggregation ([#8801](https://github.com/opensearch-project/OpenSearch/pull/8801))
 - [Remove] Deprecated Fractional ByteSizeValue support #9005 ([#9005](https://github.com/opensearch-project/OpenSearch/pull/9005))
 - Make MultiBucketConsumerService thread safe to use across slices during search ([#9047](https://github.com/opensearch-project/OpenSearch/pull/9047))

buildSrc/src/main/java/org/opensearch/gradle/http/WaitForHttpResource.java

@@ -83,6 +83,24 @@ public WaitForHttpResource(String protocol, String host, int numberOfNodes) thro
         this(new URL(protocol + "://" + host + "/_cluster/health?wait_for_nodes=>=" + numberOfNodes + "&wait_for_status=yellow"));
     }
 
+    public WaitForHttpResource(String protocol, String host, String username, String password, int numberOfNodes)
+        throws MalformedURLException {
+        this(
+            new URL(
+                protocol
+                    + "://"
+                    + username
+                    + ":"
+                    + password
+                    + "@"
+                    + host
+                    + "/_cluster/health?wait_for_nodes=>="
+                    + numberOfNodes
+                    + "&wait_for_status=yellow"
+            )
+        );
+    }
+
     public WaitForHttpResource(URL url) {
         this.url = url;
     }

buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchCluster.java

@@ -81,7 +81,6 @@ public class OpenSearchCluster implements TestClusterConfiguration, Named {
     private final FileSystemOperations fileSystemOperations;
     private final ArchiveOperations archiveOperations;
     private int nodeIndex = 0;
-
     private int zoneCount = 1;
 
     public OpenSearchCluster(
@@ -100,7 +99,6 @@ public OpenSearchCluster(
         this.archiveOperations = archiveOperations;
         this.workingDirBase = workingDirBase;
         this.nodes = project.container(OpenSearchNode.class);
-
         // Always add the first node
         String zone = hasZoneProperty() ? "zone-1" : "";
         addNode(clusterName + "-0", zone);
@@ -265,6 +263,11 @@ public void keystorePassword(String password) {
         nodes.all(each -> each.keystorePassword(password));
     }
 
+    @Override
+    public void setSecure(boolean secure) {
+        nodes.all(each -> each.setSecure(secure));
+    }
+
     @Override
     public void cliSetup(String binTool, CharSequence... args) {
         nodes.all(each -> each.cliSetup(binTool, args));
@@ -367,6 +370,7 @@ private void commonNodeConfig() {
         } else {
             nodeNames = nodes.stream().map(OpenSearchNode::getName).map(this::safeName).collect(Collectors.joining(","));
         }
+
         OpenSearchNode firstNode = null;
         for (OpenSearchNode node : nodes) {
             // Can only configure master nodes if we have node names defined
@@ -554,12 +558,25 @@ public OpenSearchNode singleNode() {
     private void addWaitForClusterHealth() {
         waitConditions.put("cluster health yellow", (node) -> {
             try {
-                WaitForHttpResource wait = new WaitForHttpResource("http", getFirstNode().getHttpSocketURI(), nodes.size());
-
-                List<Map<String, String>> credentials = getFirstNode().getCredentials();
-                if (getFirstNode().getCredentials().isEmpty() == false) {
-                    wait.setUsername(credentials.get(0).get("useradd"));
-                    wait.setPassword(credentials.get(0).get("-p"));
+                WaitForHttpResource wait;
+                if (!getFirstNode().isSecure()) {
+                    wait = new WaitForHttpResource("http", getFirstNode().getHttpSocketURI(), nodes.size());
+                    List<Map<String, String>> credentials = getFirstNode().getCredentials();
+                    if (getFirstNode().getCredentials().isEmpty() == false) {
+                        wait.setUsername(credentials.get(0).get("useradd"));
+                        wait.setPassword(credentials.get(0).get("-p"));
+                    }
+                } else {
+                    wait = new WaitForHttpResource(
+                        "https",
+                        getFirstNode().getHttpSocketURI(),
+                        getFirstNode().getCredentials().get(0).get("username"),
+                        getFirstNode().getCredentials().get(0).get("password"),
+                        nodes.size()
+                    );
+                    wait.setUsername(getFirstNode().getCredentials().get(0).get("username"));
+                    wait.setPassword(getFirstNode().getCredentials().get(0).get("password"));
+                    wait.setCertificateAuthorities(getFirstNode().getExtraConfigFilesMap().get("root-ca.pem"));
                 }
                 return wait.wait(500);
             } catch (IOException e) {

buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java

@@ -160,6 +160,7 @@ public class OpenSearchNode implements TestClusterConfiguration {
     private final Path httpPortsFile;
     private final Path tmpDir;
 
+    private boolean secure = false;
     private int currentDistro = 0;
     private TestDistribution testDistribution;
     private final List<OpenSearchDistribution> distributions = new ArrayList<>();
@@ -209,6 +210,7 @@ public class OpenSearchNode implements TestClusterConfiguration {
         setTestDistribution(TestDistribution.INTEG_TEST);
         setVersion(VersionProperties.getOpenSearch());
         this.zone = zone;
+        this.credentials.add(new HashMap<>());
     }
 
     @Input
@@ -217,6 +219,11 @@ public String getName() {
         return nameCustomization.apply(name);
     }
 
+    @Internal
+    public boolean isSecure() {
+        return secure;
+    }
+
     @Internal
     public Version getVersion() {
         return Version.fromString(distributions.get(currentDistro).getVersion());
@@ -452,6 +459,11 @@ public void setPreserveDataDir(boolean preserveDataDir) {
         this.preserveDataDir = preserveDataDir;
     }
 
+    @Override
+    public void setSecure(boolean secure) {
+        this.secure = secure;
+    }
+
     @Override
     public void freeze() {
         requireNonNull(testDistribution, "null testDistribution passed when configuring test cluster `" + this + "`");
@@ -471,6 +483,17 @@ public Stream<String> logLines() throws IOException {
     @Override
     public synchronized void start() {
         LOGGER.info("Starting `{}`", this);
+        if (System.getProperty("tests.opensearch.secure").equalsIgnoreCase("true")) {
+            secure = Boolean.getBoolean(System.getProperty("tests.opensearch.secure"));
+        }
+        if (System.getProperty("tests.opensearch.username") != null) {
+            this.credentials.get(0).put("username", System.getProperty("tests.opensearch.username"));
+            LOGGER.info("Overwriting username to: " + this.getCredentials().get(0).get("username"));
+        }
+        if (System.getProperty("tests.opensearch.password") != null) {
+            this.credentials.get(0).put("password", System.getProperty("tests.opensearch.password"));
+            LOGGER.info("Overwriting password to: " + this.getCredentials().get(0).get("password"));
+        }
         if (Files.exists(getExtractedDistributionDir()) == false) {
             throw new TestClustersException("Can not start " + this + ", missing: " + getExtractedDistributionDir());
         }
@@ -1349,6 +1372,11 @@ public List<?> getExtraConfigFiles() {
         return extraConfigFiles.getNormalizedCollection();
     }
 
+    @Internal
+    public Map<String, File> getExtraConfigFilesMap() {
+        return extraConfigFiles;
+    }
+
     @Override
     @Internal
     public boolean isProcessAlive() {

buildSrc/src/main/java/org/opensearch/gradle/testclusters/TestClusterConfiguration.java

@@ -108,6 +108,8 @@ public interface TestClusterConfiguration {
 
     void setPreserveDataDir(boolean preserveDataDir);
 
+    void setSecure(boolean secure);
+
     void freeze();
 
     void start();