CLI tool to reconfigure nodes to enroll (#79690)

This change introduces a CLI tool that can be run directly after
installation time in packaged installations, to allow for a node
that was auto-configured to be the initial node of a cluster during
installation ( default installation behavior) to be reconfigured
to join an existing cluster, using an enrollment token.
The use of this tool presumes that the user has the
appropriate permissions to read/write to the installation dirs and
that this node has not been yet started, i.e. this tool is run
directly after installation. It is destructive, as it removes
existing security auto-configuration, and as such it requires an
explicit verification from the user.

This is a follow-up to #7718.

distribution/packages/src/common/scripts/postinst

@@ -79,6 +79,9 @@ if [ "x$IS_UPGRADE" != "xtrue" ]; then
             echo
             echo "The generated password for the elastic built-in superuser is : ${INITIAL_PASSWORD}"
             echo
+            echo "If this node should join an existing cluster, you can reconfigure this with"
+            echo "'/usr/share/elasticsearch/bin/elasticsearch-reconfigure-node --enrollment-token <token-here>'"
+            echo "after creating an enrollment token on your existing cluster."
             echo
             echo "You can complete the following actions at any time:"
             echo

docs/changelog/79690.yaml

@@ -0,0 +1,5 @@
+pr: 79690
+summary: CLI tool to reconfigure nodes to enroll
+area: "Security"
+type: enhancement
+issues: []

qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java

@@ -234,7 +234,7 @@ public void test50AutoConfigurationFailsWhenCertificatesNotGenerated() throws Ex
             tempDir.resolve("bcprov-jdk15on-1.64.jar")
         );
         Shell.Result result = runElasticsearchStartCommand(null, false, false);
-        assertElasticsearchFailure(result, "java.lang.NoClassDefFoundError: org/bouncycastle/asn1/x509/GeneralName", null);
+        assertElasticsearchFailure(result, "java.lang.NoClassDefFoundError: org/bouncycastle/", null);
         Files.move(
             tempDir.resolve("bcprov-jdk15on-1.64.jar"),
             installation.lib.resolve("tools").resolve("security-cli").resolve("bcprov-jdk15on-1.64.jar")

qa/os/src/test/java/org/elasticsearch/packaging/test/PackagesSecurityAutoConfigurationTests.java

@@ -8,8 +8,10 @@
 
 package org.elasticsearch.packaging.test;
 
+import org.elasticsearch.cli.ExitCodes;
 import org.elasticsearch.packaging.util.Installation;
 import org.elasticsearch.packaging.util.Packages;
+import org.elasticsearch.packaging.util.Shell;
 import org.junit.BeforeClass;
 
 import java.nio.file.Files;
@@ -19,11 +21,13 @@
 import java.util.Optional;
 import java.util.function.Predicate;
 
+import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING;
 import static org.elasticsearch.packaging.util.FileUtils.append;
 import static org.elasticsearch.packaging.util.Packages.assertInstalled;
 import static org.elasticsearch.packaging.util.Packages.assertRemoved;
 import static org.elasticsearch.packaging.util.Packages.installPackage;
 import static org.elasticsearch.packaging.util.Packages.verifyPackageInstallation;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.is;
@@ -95,6 +99,145 @@ public void test40SecurityNotAutoConfiguredWhenExistingKeystoreUnknownPassword()
         assertThat(configLines, not(hasItem("# have been automatically generated in order to configure Security.               #")));
     }
 
+    public void test50ReconfigureAndEnroll() throws Exception {
+        cleanup();
+        assertRemoved(distribution());
+        installation = installPackage(sh, distribution(), successfulAutoConfiguration());
+        assertInstalled(distribution());
+        verifyPackageInstallation(installation, distribution(), sh);
+        verifySecurityAutoConfigured(installation);
+        assertNotNull(installation.getElasticPassword());
+        // We cannot run two packaged installations simultaneously here so that we can test that the second node enrolls successfully
+        // We trigger with an invalid enrollment token, to verify that we removed the existing auto-configuration
+        Shell.Result result = installation.executables().nodeReconfigureTool.run("--enrollment-token thisisinvalid", "y", true);
+        assertThat(result.exitCode, equalTo(ExitCodes.DATA_ERROR)); // invalid enrollment token
+        verifySecurityNotAutoConfigured(installation);
+    }
+
+    public void test60ReconfigureWithoutEnrollmentToken() throws Exception {
+        cleanup();
+        assertRemoved(distribution());
+        installation = installPackage(sh, distribution(), successfulAutoConfiguration());
+        assertInstalled(distribution());
+        verifyPackageInstallation(installation, distribution(), sh);
+        verifySecurityAutoConfigured(installation);
+        assertNotNull(installation.getElasticPassword());
+        Shell.Result result = installation.executables().nodeReconfigureTool.run("", null, true);
+        assertThat(result.exitCode, equalTo(ExitCodes.USAGE)); // missing enrollment token
+        // we fail on command invocation so we don't even try to remove autoconfiguration
+        verifySecurityAutoConfigured(installation);
+    }
+
+    // The following could very well be unit tests but the way we delete files doesn't play well with jimfs
+
+    public void test70ReconfigureFailsWhenTlsAutoConfDirMissing() throws Exception {
+        cleanup();
+        assertRemoved(distribution());
+        installation = installPackage(sh, distribution(), successfulAutoConfiguration());
+        assertInstalled(distribution());
+        verifyPackageInstallation(installation, distribution(), sh);
+        verifySecurityAutoConfigured(installation);
+        assertNotNull(installation.getElasticPassword());
+
+        Optional<String> autoConfigDirName = getAutoConfigDirName(installation);
+        // Move instead of delete because Files.deleteIfExists bails on non empty dirs
+        Files.move(installation.config(autoConfigDirName.get()), installation.config("temp-autoconf-dir"));
+        Shell.Result result = installation.executables().nodeReconfigureTool.run("--enrollment-token a-token", "y", true);
+        assertThat(result.exitCode, equalTo(ExitCodes.USAGE)); //
+    }
+
+    public void test71ReconfigureFailsWhenKeyStorePasswordWrong() throws Exception {
+        cleanup();
+        assertRemoved(distribution());
+        installation = installPackage(sh, distribution(), successfulAutoConfiguration());
+        assertInstalled(distribution());
+        verifyPackageInstallation(installation, distribution(), sh);
+        verifySecurityAutoConfigured(installation);
+        assertNotNull(installation.getElasticPassword());
+        Shell.Result changePassword = installation.executables().keystoreTool.run(
+            "passwd",
+            "some-password" + "\n" + "some-password" + "\n"
+        );
+        assertThat(changePassword.exitCode, equalTo(0));
+        Shell.Result result = installation.executables().nodeReconfigureTool.run(
+            "--enrollment-token a-token",
+            "y" + "\n" + "some-wrong-password",
+            true
+        );
+        assertThat(result.exitCode, equalTo(ExitCodes.IO_ERROR)); //
+        assertThat(result.stderr, containsString("Error was: Provided keystore password was incorrect"));
+    }
+
+    public void test71ReconfigureFailsWhenKeyStoreDoesNotContainExpectedSettings() throws Exception {
+        cleanup();
+        assertRemoved(distribution());
+        installation = installPackage(sh, distribution(), successfulAutoConfiguration());
+        assertInstalled(distribution());
+        verifyPackageInstallation(installation, distribution(), sh);
+        verifySecurityAutoConfigured(installation);
+        assertNotNull(installation.getElasticPassword());
+        Shell.Result removeSetting = installation.executables().keystoreTool.run(
+            "remove xpack.security.transport.ssl.keystore.secure_password"
+        );
+        assertThat(removeSetting.exitCode, equalTo(0));
+        Shell.Result result = installation.executables().nodeReconfigureTool.run("--enrollment-token a-token", "y", true);
+        assertThat(result.exitCode, equalTo(ExitCodes.IO_ERROR));
+        assertThat(
+            result.stderr,
+            containsString(
+                "elasticsearch.keystore did not contain expected setting [xpack.security.transport.ssl.keystore.secure_password]."
+            )
+        );
+    }
+
+    public void test72ReconfigureFailsWhenConfigurationDoesNotContainSecurityAutoConfig() throws Exception {
+        cleanup();
+        assertRemoved(distribution());
+        installation = installPackage(sh, distribution(), successfulAutoConfiguration());
+        assertInstalled(distribution());
+        verifyPackageInstallation(installation, distribution(), sh);
+        verifySecurityAutoConfigured(installation);
+        assertNotNull(installation.getElasticPassword());
+        // We remove everything. We don't need to be precise and remove only auto-configuration, the rest are commented out either way
+        Path yml = installation.config("elasticsearch.yml");
+        Files.write(yml, List.of(), TRUNCATE_EXISTING);
+
+        Shell.Result result = installation.executables().nodeReconfigureTool.run("--enrollment-token a-token", "y", true);
+        assertThat(result.exitCode, equalTo(ExitCodes.USAGE)); //
+        assertThat(result.stderr, containsString("Expected configuration is missing from elasticsearch.yml."));
+    }
+
+    public void test72ReconfigureRetainsUserSettings() throws Exception {
+        cleanup();
+        assertRemoved(distribution());
+        installation = installPackage(sh, distribution(), successfulAutoConfiguration());
+        assertInstalled(distribution());
+        verifyPackageInstallation(installation, distribution(), sh);
+        verifySecurityAutoConfigured(installation);
+        assertNotNull(installation.getElasticPassword());
+        // We remove everything. We don't need to be precise and remove only auto-configuration, the rest are commented out either way
+        Path yml = installation.config("elasticsearch.yml");
+        List<String> allLines = Files.readAllLines(yml);
+        // Replace a comment we know exists in the auto-configuration stanza, with a user defined setting
+        allLines.set(
+            allLines.indexOf("# All the nodes use the same key and certificate on the inter-node connection"),
+            "cluster.name: testclustername"
+        );
+        allLines.add("node.name: testnodename");
+        Files.write(yml, allLines, TRUNCATE_EXISTING);
+
+        // We cannot run two packaged installations simultaneously here so that we can test that the second node enrolls successfully
+        // We trigger with an invalid enrollment token, to verify that we removed the existing auto-configuration
+        Shell.Result result = installation.executables().nodeReconfigureTool.run("--enrollment-token thisisinvalid", "y", true);
+        assertThat(result.exitCode, equalTo(ExitCodes.DATA_ERROR)); // invalid enrollment token
+        verifySecurityNotAutoConfigured(installation);
+        // Check that user configuration , both inside and outside the autocofiguration stanza, was retained
+        Path editedYml = installation.config("elasticsearch.yml");
+        List<String> newConfigurationLines = Files.readAllLines(editedYml);
+        assertThat(newConfigurationLines, hasItem("cluster.name: testclustername"));
+        assertThat(newConfigurationLines, hasItem("node.name: testnodename"));
+    }
+
     private Predicate<String> successfulAutoConfiguration() {
         Predicate<String> p1 = output -> output.contains("Authentication and authorization are enabled.");
         Predicate<String> p2 = output -> output.contains("TLS for the transport and HTTP layers is enabled and configured.");

qa/os/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java

@@ -723,10 +723,12 @@ public void verifySecurityAutoConfigured(Installation es) throws Exception {
     public static void verifySecurityNotAutoConfigured(Installation es) throws Exception {
         assertThat(getAutoConfigDirName(es).isPresent(), Matchers.is(false));
         if (es.distribution.isPackage()) {
-            assertThat(
-                sh.run(es.executables().keystoreTool + " list").stdout,
-                not(Matchers.containsString("autoconfiguration.password_hash"))
-            );
+            if (Files.exists(es.config("elasticsearch.keystore"))) {
+                assertThat(
+                    sh.run(es.executables().keystoreTool + " list").stdout,
+                    not(Matchers.containsString("autoconfiguration.password_hash"))
+                );
+            }
         }
         List<String> configLines = Files.readAllLines(es.config("elasticsearch.yml"));
         assertThat(configLines, not(contains(containsString("automatically generated in order to configure Security"))));

qa/os/src/test/java/org/elasticsearch/packaging/util/Installation.java

@@ -171,6 +171,10 @@ public Shell.Result run(String args) {
         }
 
         public Shell.Result run(String args, String input) {
+            return run(args, input, false);
+        }
+
+        public Shell.Result run(String args, String input, boolean ignoreExitCode) {
             String command = path.toString();
             if (Platforms.WINDOWS) {
                 command = "& '" + command + "'";
@@ -184,6 +188,9 @@ public Shell.Result run(String args, String input) {
             if (input != null) {
                 command = "echo \"" + input + "\" | " + command;
             }
+            if (ignoreExitCode) {
+                return sh.runIgnoreExitCode(command + " " + args);
+            }
             return sh.run(command + " " + args);
         }
     }
@@ -201,6 +208,7 @@ public class Executables {
         public final Executable setupPasswordsTool = new Executable("elasticsearch-setup-passwords");
         public final Executable resetPasswordTool = new Executable("elasticsearch-reset-password");
         public final Executable createEnrollmentToken = new Executable("elasticsearch-create-enrollment-token");
+        public final Executable nodeReconfigureTool = new Executable("elasticsearch-reconfigure-node");
         public final Executable sqlCli = new Executable("elasticsearch-sql-cli");
         public final Executable syskeygenTool = new Executable("elasticsearch-syskeygen");
         public final Executable usersTool = new Executable("elasticsearch-users");