Allow node to enroll to cluster on startup (#77718)

The functionality to enroll a new node to a cluster was
introduced in #77292 as a CLI tool. This change replaces this
CLI tool with the option to trigger the enrollment functionality 
on startup of elasticsearch via a named argument that can be 
passed to the elasticsearch startup script (--enrollment-token)
so that the users that want to enroll a node to a cluster can do 
this with one command instead of two. 

In a followup PR we are introducing a CLI tool version of this
functionality, that can be used to reconfigure packaged
installations.

client/rest-high-level/qa/ssl-enabled/src/javaRestTest/java/org/elasticsearch/client/EnrollmentIT.java

@@ -76,6 +76,7 @@ public void testEnrollNode() throws Exception {
         assertThat(nodeEnrollmentResponse, notNullValue());
         assertThat(nodeEnrollmentResponse.getHttpCaKey(), endsWith("K2S3vidA="));
         assertThat(nodeEnrollmentResponse.getHttpCaCert(), endsWith("LfkRjirc="));
+        assertThat(nodeEnrollmentResponse.getTransportCaCert(), endsWith("3J9+kpgIbE"));
         assertThat(nodeEnrollmentResponse.getTransportKey(), endsWith("1I+r8vOQ=="));
         assertThat(nodeEnrollmentResponse.getTransportCert(), endsWith("OpTdtgJo="));
         List<String> nodesAddresses = nodeEnrollmentResponse.getNodesAddresses();

client/rest-high-level/qa/ssl-enabled/src/javaRestTest/java/org/elasticsearch/client/documentation/EnrollmentDocumentationIT.java

@@ -75,9 +75,10 @@ public void testNodeEnrollment() throws Exception {
             // tag::node-enrollment-response
             String httpCaKey = response.getHttpCaKey(); // <1>
             String httpCaCert = response.getHttpCaCert(); // <2>
-            String transportKey = response.getTransportKey(); // <3>
-            String transportCert = response.getTransportCert(); // <4>
-            List<String> nodesAddresses = response.getNodesAddresses();  // <5>
+            String transportCaCert = response.getTransportCaCert(); // <3>
+            String transportKey = response.getTransportKey(); // <4>
+            String transportCert = response.getTransportCert(); // <5>
+            List<String> nodesAddresses = response.getNodesAddresses();  // <6>
             // end::node-enrollment-response
         }
 

client/rest-high-level/src/main/java/org/elasticsearch/client/security/NodeEnrollmentResponse.java

@@ -21,14 +21,16 @@ public class NodeEnrollmentResponse {
 
     private final String httpCaKey;
     private final String httpCaCert;
+    private final String transportCaCert;
     private final String transportKey;
     private final String transportCert;
     private final List<String> nodesAddresses;
 
-    public NodeEnrollmentResponse(String httpCaKey, String httpCaCert, String transportKey, String transportCert,
+    public NodeEnrollmentResponse(String httpCaKey, String httpCaCert, String transportCaCert, String transportKey, String transportCert,
                                   List<String> nodesAddresses){
         this.httpCaKey = httpCaKey;
         this.httpCaCert = httpCaCert;
+        this.transportCaCert = transportCaCert;
         this.transportKey = transportKey;
         this.transportCert = transportCert;
         this.nodesAddresses = Collections.unmodifiableList(nodesAddresses);
@@ -46,6 +48,10 @@ public String getTransportKey() {
         return transportKey;
     }
 
+    public String getTransportCaCert() {
+        return transportCaCert;
+    }
+
     public String getTransportCert() {
         return transportCert;
     }
@@ -56,6 +62,7 @@ public List<String> getNodesAddresses() {
 
     private static final ParseField HTTP_CA_KEY = new ParseField("http_ca_key");
     private static final ParseField HTTP_CA_CERT = new ParseField("http_ca_cert");
+    private static final ParseField TRANSPORT_CA_CERT = new ParseField("transport_ca_cert");
     private static final ParseField TRANSPORT_KEY = new ParseField("transport_key");
     private static final ParseField TRANSPORT_CERT = new ParseField("transport_cert");
     private static final ParseField NODES_ADDRESSES = new ParseField("nodes_addresses");
@@ -66,15 +73,17 @@ public List<String> getNodesAddresses() {
         new ConstructingObjectParser<>(NodeEnrollmentResponse.class.getName(), true, a -> {
             final String httpCaKey = (String) a[0];
             final String httpCaCert = (String) a[1];
-            final String transportKey = (String) a[2];
-            final String transportCert = (String) a[3];
-            final List<String> nodesAddresses = (List<String>) a[4];
-            return new NodeEnrollmentResponse(httpCaKey, httpCaCert, transportKey, transportCert, nodesAddresses);
+            final String transportCaCert = (String) a[2];
+            final String transportKey = (String) a[3];
+            final String transportCert = (String) a[4];
+            final List<String> nodesAddresses = (List<String>) a[5];
+            return new NodeEnrollmentResponse(httpCaKey, httpCaCert, transportCaCert, transportKey, transportCert, nodesAddresses);
         });
 
     static {
         PARSER.declareString(ConstructingObjectParser.constructorArg(), HTTP_CA_KEY);
         PARSER.declareString(ConstructingObjectParser.constructorArg(), HTTP_CA_CERT);
+        PARSER.declareString(ConstructingObjectParser.constructorArg(), TRANSPORT_CA_CERT);
         PARSER.declareString(ConstructingObjectParser.constructorArg(), TRANSPORT_KEY);
         PARSER.declareString(ConstructingObjectParser.constructorArg(), TRANSPORT_CERT);
         PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), NODES_ADDRESSES);
@@ -88,12 +97,15 @@ public static NodeEnrollmentResponse fromXContent(XContentParser parser) throws
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
         NodeEnrollmentResponse that = (NodeEnrollmentResponse) o;
-        return httpCaKey.equals(that.httpCaKey) && httpCaCert.equals(that.httpCaCert) && transportKey.equals(that.transportKey)
+        return httpCaKey.equals(that.httpCaKey)
+            && httpCaCert.equals(that.httpCaCert)
+            && transportCaCert.equals(that.transportCaCert)
+            && transportKey.equals(that.transportKey)
             && transportCert.equals(that.transportCert)
             && nodesAddresses.equals(that.nodesAddresses);
     }
 
     @Override public int hashCode() {
-        return Objects.hash(httpCaKey, httpCaCert, transportKey, transportCert, nodesAddresses);
+        return Objects.hash(httpCaKey, httpCaCert, transportCaCert, transportKey, transportCert, nodesAddresses);
     }
 }

distribution/packages/src/common/scripts/postinst

@@ -57,12 +57,12 @@ if [ "x$IS_UPGRADE" != "xtrue" ]; then
     # Don't exit immediately on error, we want to hopefully print some helpful banners
     set +e
     # Attempt to auto-configure security, this seems to be an installation
-    if ES_MAIN_CLASS=org.elasticsearch.xpack.security.cli.ConfigInitialNode \
+    if ES_MAIN_CLASS=org.elasticsearch.xpack.security.cli.AutoConfigureNode \
     ES_ADDITIONAL_SOURCES="x-pack-env;x-pack-security-env" \
     ES_ADDITIONAL_CLASSPATH_DIRECTORIES=lib/tools/security-cli \
     /usr/share/elasticsearch/bin/elasticsearch-cli <<< ""; then
         # Above command runs as root and TLS keystores are created group-owned by root. It's simple to correct the ownership here
-        for dir in "${ES_PATH_CONF}"/tls_auto_config_initial_node_*
+        for dir in "${ES_PATH_CONF}"/tls_auto_config_*
         do
             chown root:elasticsearch "${dir}"/http_keystore_local_node.p12
             chown root:elasticsearch "${dir}"/http_ca.crt
@@ -83,13 +83,13 @@ if [ "x$IS_UPGRADE" != "xtrue" ]; then
             echo "You can complete the following actions at any time:"
             echo
             echo "Reset the password of the elastic built-in superuser with "
-            echo "'/usr/share/bin/elasticsearch-reset-password -u elastic'."
+            echo "'/usr/share/elasticsearch/bin/elasticsearch-reset-password -u elastic'."
             echo
             echo "Generate an enrollment token for Kibana instances with "
-            echo " 'bin/elasticsearch-create-enrollment-token -s kibana'."
+            echo " '/usr/share/elasticsearch/bin/elasticsearch-create-enrollment-token -s kibana'."
             echo
             echo "Generate an enrollment token for Elasticsearch nodes with "
-            echo "'bin/elasticsearch-create-enrollment-token -s node'."
+            echo "'/usr/share/elasticsearch/bin/elasticsearch-create-enrollment-token -s node'."
             echo
             echo "-------------------------------------------------------------------------------------------------"
         fi
@@ -108,7 +108,7 @@ if [ "x$IS_UPGRADE" != "xtrue" ]; then
             echo "However, authentication and authorization are still enabled."
             echo
             echo "You can reset the password of the elastic built-in superuser with "
-            echo "'/usr/share/bin/elasticsearch-reset-password -u elastic' at any time."
+            echo "'/usr/share/elasticsearch/bin/elasticsearch-reset-password -u elastic' at any time."
             echo "-------------------------------------------------------------------------------------------------"
         fi
     fi

distribution/packages/src/common/scripts/postrm

@@ -103,7 +103,7 @@ if [ "$REMOVE_DIRS" = "true" ]; then
 
     # delete the security auto config directory if we are purging
     if [ "$REMOVE_SECURITY_AUTO_CONFIG_DIRECTORY" = "true" ]; then
-      for dir in "${ES_PATH_CONF}"/tls_auto_config_initial_node_*
+      for dir in "${ES_PATH_CONF}"/tls_auto_config_*
       do
         echo -n "Deleting security auto-configuration directory..."
         rm -rf "${dir}"

distribution/src/bin/elasticsearch

@@ -18,16 +18,29 @@ source "`dirname "$0"`"/elasticsearch-env
 CHECK_KEYSTORE=true
 ATTEMPT_SECURITY_AUTO_CONFIG=true
 DAEMONIZE=false
-for option in "$@"; do
-  case "$option" in
-    -h|--help|-V|--version)
-      CHECK_KEYSTORE=false
-      ATTEMPT_SECURITY_AUTO_CONFIG=false
-      ;;
-    -d|--daemonize)
-      DAEMONIZE=true
-      ;;
-  esac
+ENROLL_TO_CLUSTER=false
+# Store original arg array as we will be shifting through it below
+ARG_LIST=("$@")
+
+while [ $# -gt 0 ]; do
+   if [[ $1 == "--enrollment-token" ]]; then
+     if [ $ENROLL_TO_CLUSTER = true ]; then
+       echo "Multiple --enrollment-token parameters are not allowed" 1>&2
+       exit 1
+     fi
+     ENROLL_TO_CLUSTER=true
+     ATTEMPT_SECURITY_AUTO_CONFIG=false
+     ENROLLMENT_TOKEN="$2"
+     shift
+   elif [[ $1 == "-h" || $1 == "--help" || $1 == "-V" || $1 == "--version" ]]; then
+     CHECK_KEYSTORE=false
+     ATTEMPT_SECURITY_AUTO_CONFIG=false
+   elif [[ $1 == "-d" || $1 == "--daemonize" ]]; then
+     DAEMONIZE=true
+   fi
+   if [[ $# -gt 0 ]]; then
+     shift
+   fi
 done
 
 if [ -z "$ES_TMPDIR" ]; then
@@ -47,16 +60,21 @@ then
   fi
 fi
 
-if [[ $ATTEMPT_SECURITY_AUTO_CONFIG = true ]]; then
+if [[ $ENROLL_TO_CLUSTER = true ]]; then
+  ES_MAIN_CLASS=org.elasticsearch.xpack.security.cli.AutoConfigureNode \
+  ES_ADDITIONAL_SOURCES="x-pack-env;x-pack-security-env" \
+  ES_ADDITIONAL_CLASSPATH_DIRECTORIES=lib/tools/security-cli \
+  bin/elasticsearch-cli "${ARG_LIST[@]}" <<<"$KEYSTORE_PASSWORD"
+elif [[ $ATTEMPT_SECURITY_AUTO_CONFIG = true ]]; then
   # It is possible that an auto-conf failure prevents the node from starting, but this is only the exceptional case (exit code 1).
   # Most likely an auto-conf failure will leave the configuration untouched (exit codes 73, 78 and 80), optionally printing a message
   # if the error is uncommon or unexpected, but it should otherwise let the node to start as usual.
   # It is passed in all the command line options in order to read the node settings ones (-E), while the other parameters are ignored
   # (a small caveat is that it also inspects the -v option in order to provide more information on how auto config went)
-  if ES_MAIN_CLASS=org.elasticsearch.xpack.security.cli.ConfigInitialNode \
+  if ES_MAIN_CLASS=org.elasticsearch.xpack.security.cli.AutoConfigureNode \
     ES_ADDITIONAL_SOURCES="x-pack-env;x-pack-security-env" \
     ES_ADDITIONAL_CLASSPATH_DIRECTORIES=lib/tools/security-cli \
-    bin/elasticsearch-cli "$@" <<<"$KEYSTORE_PASSWORD"; then
+    bin/elasticsearch-cli "${ARG_LIST[@]}" <<<"$KEYSTORE_PASSWORD"; then
       :
   else
     retval=$?
@@ -77,6 +95,13 @@ fi
 #   - fourth, ergonomic JVM options are applied
 ES_JAVA_OPTS=`export ES_TMPDIR; "$JAVA" "$XSHARE" -cp "$ES_CLASSPATH" org.elasticsearch.tools.launchers.JvmOptionsParser "$ES_PATH_CONF" "$ES_HOME/plugins"`
 
+# Remove enrollment related parameters before passing the arg list to Elasticsearch
+for i in "${!ARG_LIST[@]}"; do
+  if [[ ${ARG_LIST[i]} = "--enrollment-token" || ${ARG_LIST[i]} = "$ENROLLMENT_TOKEN" ]]; then
+    unset 'ARG_LIST[i]'
+  fi
+done
+
 # manual parsing to find out, if process should be detached
 if [[ $DAEMONIZE = false ]]; then
   exec \
@@ -90,7 +115,7 @@ if [[ $DAEMONIZE = false ]]; then
     -Des.bundled_jdk="$ES_BUNDLED_JDK" \
     -cp "$ES_CLASSPATH" \
     org.elasticsearch.bootstrap.Elasticsearch \
-    "$@" <<<"$KEYSTORE_PASSWORD"
+    "${ARG_LIST[@]}" <<<"$KEYSTORE_PASSWORD"
 else
   exec \
     "$JAVA" \
@@ -103,7 +128,7 @@ else
     -Des.bundled_jdk="$ES_BUNDLED_JDK" \
     -cp "$ES_CLASSPATH" \
     org.elasticsearch.bootstrap.Elasticsearch \
-    "$@" \
+    "${ARG_LIST[@]}" \
     <<<"$KEYSTORE_PASSWORD" &
   retval=$?
   pid=$!

distribution/src/bin/elasticsearch.bat

@@ -5,14 +5,15 @@ setlocal enableextensions
 
 SET params='%*'
 SET checkpassword=Y
+SET enrolltocluster=N
 SET attemptautoconfig=Y
 
 :loop
 FOR /F "usebackq tokens=1* delims= " %%A IN (!params!) DO (
+    SET previous=!current!
     SET current=%%A
     SET params='%%B'
 	SET silent=N
-
 	IF "!current!" == "-s" (
 		SET silent=Y
 	)
@@ -38,14 +39,33 @@ FOR /F "usebackq tokens=1* delims= " %%A IN (!params!) DO (
 		SET attemptautoconfig=N
 	)
 
+	IF "!current!" == "--enrollment-token" (
+	    IF "!enrolltocluster!" == "Y" (
+	        ECHO "Multiple --enrollment-token parameters are not allowed" 1>&2
+	        goto exitwithone
+	    )
+		SET enrolltocluster=Y
+		SET attemptautoconfig=N
+	)
+
+	IF "!previous!" == "--enrollment-token" (
+		SET enrollmenttoken="!current!"
+	)
+
 	IF "!silent!" == "Y" (
 		SET nopauseonerror=Y
 	) ELSE (
-	    IF "x!newparams!" NEQ "x" (
-	        SET newparams=!newparams! !current!
-        ) ELSE (
-            SET newparams=!current!
-        )
+	    SET SHOULD_SKIP=false
+		IF "!previous!" == "--enrollment-token" SET SHOULD_SKIP=true
+		IF "!current!" == "--enrollment-token" SET SHOULD_SKIP=true
+		IF "!SHOULD_SKIP!" == "false" (
+			IF "x!newparams!" NEQ "x" (
+				SET newparams=!newparams! !current!
+			) ELSE (
+				SET newparams=!current!
+			)
+		)
+
 	)
 
     IF "x!params!" NEQ "x" (
@@ -73,13 +93,21 @@ IF "%checkpassword%"=="Y" (
   )
 )
 
+rem windows batch pipe will choke on special characters in strings
+SET KEYSTORE_PASSWORD=!KEYSTORE_PASSWORD:^^=^^^^!
+SET KEYSTORE_PASSWORD=!KEYSTORE_PASSWORD:^&=^^^&!
+SET KEYSTORE_PASSWORD=!KEYSTORE_PASSWORD:^|=^^^|!
+SET KEYSTORE_PASSWORD=!KEYSTORE_PASSWORD:^<=^^^<!
+SET KEYSTORE_PASSWORD=!KEYSTORE_PASSWORD:^>=^^^>!
+SET KEYSTORE_PASSWORD=!KEYSTORE_PASSWORD:^\=^^^\!
+
 IF "%attemptautoconfig%"=="Y" (
     ECHO.!KEYSTORE_PASSWORD!| %JAVA% %ES_JAVA_OPTS% ^
       -Des.path.home="%ES_HOME%" ^
       -Des.path.conf="%ES_PATH_CONF%" ^
       -Des.distribution.flavor="%ES_DISTRIBUTION_FLAVOR%" ^
       -Des.distribution.type="%ES_DISTRIBUTION_TYPE%" ^
-      -cp "!ES_CLASSPATH!;!ES_HOME!/lib/tools/security-cli/*;!ES_HOME!/modules/x-pack-core/*;!ES_HOME!/modules/x-pack-security/*" "org.elasticsearch.xpack.security.cli.ConfigInitialNode" !newparams!
+      -cp "!ES_CLASSPATH!;!ES_HOME!/lib/tools/security-cli/*;!ES_HOME!/modules/x-pack-core/*;!ES_HOME!/modules/x-pack-security/*" "org.elasticsearch.xpack.security.cli.AutoConfigureNode" !newparams!
     SET SHOULDEXIT=Y
     IF !ERRORLEVEL! EQU 0 SET SHOULDEXIT=N
     IF !ERRORLEVEL! EQU 73 SET SHOULDEXIT=N
@@ -90,6 +118,19 @@ IF "%attemptautoconfig%"=="Y" (
     )
 )
 
+IF "!enrolltocluster!"=="Y" (
+    ECHO.!KEYSTORE_PASSWORD!| %JAVA% %ES_JAVA_OPTS% ^
+      -Des.path.home="%ES_HOME%" ^
+      -Des.path.conf="%ES_PATH_CONF%" ^
+      -Des.distribution.flavor="%ES_DISTRIBUTION_FLAVOR%" ^
+      -Des.distribution.type="%ES_DISTRIBUTION_TYPE%" ^
+      -cp "!ES_CLASSPATH!;!ES_HOME!/lib/tools/security-cli/*;!ES_HOME!/modules/x-pack-core/*;!ES_HOME!/modules/x-pack-security/*" "org.elasticsearch.xpack.security.cli.AutoConfigureNode" ^
+      !newparams! --enrollment-token %enrollmenttoken%
+	IF !ERRORLEVEL! NEQ 0 (
+	    exit /b !ERRORLEVEL!
+	)
+)
+
 if not defined ES_TMPDIR (
   for /f "tokens=* usebackq" %%a in (`CALL %JAVA% -cp "!ES_CLASSPATH!" "org.elasticsearch.tools.launchers.TempDirectory"`) do set  ES_TMPDIR=%%a
 )
@@ -111,14 +152,6 @@ if "%MAYBE_JVM_OPTIONS_PARSER_FAILED%" == "jvm_options_parser_failed" (
   exit /b 1
 )
 
-rem windows batch pipe will choke on special characters in strings
-SET KEYSTORE_PASSWORD=!KEYSTORE_PASSWORD:^^=^^^^!
-SET KEYSTORE_PASSWORD=!KEYSTORE_PASSWORD:^&=^^^&!
-SET KEYSTORE_PASSWORD=!KEYSTORE_PASSWORD:^|=^^^|!
-SET KEYSTORE_PASSWORD=!KEYSTORE_PASSWORD:^<=^^^<!
-SET KEYSTORE_PASSWORD=!KEYSTORE_PASSWORD:^>=^^^>!
-SET KEYSTORE_PASSWORD=!KEYSTORE_PASSWORD:^\=^^^\!
-
 ECHO.!KEYSTORE_PASSWORD!| %JAVA% %ES_JAVA_OPTS% -Delasticsearch ^
   -Des.path.home="%ES_HOME%" -Des.path.conf="%ES_PATH_CONF%" ^
   -Des.distribution.flavor="%ES_DISTRIBUTION_FLAVOR%" ^
@@ -129,3 +162,8 @@ ECHO.!KEYSTORE_PASSWORD!| %JAVA% %ES_JAVA_OPTS% -Delasticsearch ^
 endlocal
 endlocal
 exit /b %ERRORLEVEL%
+
+rem this hack is ugly but necessary because we can't exit with /b X from within the argument parsing loop.
+rem exit 1 (without /b) would work for powershell but it will terminate the cmd process when run in cmd
+:exitwithone
+    exit /b 1

docs/java-rest/high-level/security/enroll_node.asciidoc

@@ -33,11 +33,13 @@ include-tagged::{doc-tests}/EnrollmentDocumentationIT.java[{api}-response]
 for the HTTP layer, as a Base64 encoded string of the ASN.1 DER encoding of the key.
 <2> The CA certificate that can be used by the new node in order to sign its certificate
 for the HTTP layer, as a Base64 encoded string of the ASN.1 DER encoding of the certificate.
-<3> The private key that the node can use for  TLS for its transport layer, as a Base64
+<3> The CA certificate that is used to sign the TLS certificate for the transport layer, as
+a Base64 encoded string of the ASN.1 DER encoding of the certificate.
+<4> The private key that the node can use for  TLS for its transport layer, as a Base64
 encoded string of the ASN.1 DER encoding of the key.
-<4> The certificate that the node can use for  TLS for its transport layer, as a Base64
+<5> The certificate that the node can use for  TLS for its transport layer, as a Base64
 encoded string of the ASN.1 DER encoding of the certificate.
-<5> A list of transport addresses in the form of `host:port` for the nodes that are already
+<6> A list of transport addresses in the form of `host:port` for the nodes that are already
 members of the cluster.
 
 

qa/os/build.gradle

@@ -13,6 +13,7 @@ plugins {
 dependencies {
   testImplementation project(':server')
   testImplementation project(':libs:elasticsearch-core')
+  testImplementation(testArtifact(project(':x-pack:plugin:core')))
   testImplementation "junit:junit:${versions.junit}"
   testImplementation "org.hamcrest:hamcrest:${versions.hamcrest}"
   testImplementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"