Support configuring SSL separately for remote access port (#93334)

Though the remote access is implemented with the transport profile. Its
configuration and behaviour should not be tied to the default transport
profile like other profiles do. Users should be able to enable or
disable SSL separately for the remote access port and configure differnt
values for all SSL settings. These settings can also have different
defaults.

This PR implements the above by:
* Adds a new xpack.security.remote_cluster.ssl.enabled setting to
  control whether SSL is enabled separately for the remote access port
* The above enabled setting defaults to true (unlike the default for
  tranport SSL)
* Client auth defaults to none for the remote access port
* Separate server SSL configuration validation

The PR also moves the remote access profile to be built first for more
consistent error message.

server/src/internalClusterTest/java/org/elasticsearch/transport/RemoteClusterPortSettingsIT.java

@@ -26,8 +26,16 @@ public void testDirectlyConfiguringTransportProfileForRemoteClusterWillFailToSta
         final Settings.Builder builder = Settings.builder()
             .put(randomBoolean() ? masterNode() : dataOnlyNode())
             .put("discovery.initial_state_timeout", "1s")
-            .put("remote_cluster.enabled", true)
-            .put("transport.profiles._remote_cluster.port", 9900);
+            .put("remote_cluster.enabled", true);
+
+        // Test that the same error message is always reported for direct usage of the _remote_cluster profile
+        switch (randomIntBetween(0, 2)) {
+            case 0 -> builder.put("transport.profiles._remote_cluster.tcp.keep_alive", true);
+            case 1 -> builder.put("transport.profiles._remote_cluster.port", 9900);
+            default -> builder.put("transport.profiles._remote_cluster.port", 9900)
+                .put("transport.profiles._remote_cluster.tcp.keep_alive", true);
+        }
+
         final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> internalCluster().startNode(builder));
         assertThat(
             e.getMessage(),

server/src/main/java/org/elasticsearch/transport/TcpTransport.java

@@ -1018,6 +1018,11 @@ public final TransportStats getStats() {
      */
     public static Set<ProfileSettings> getProfileSettings(Settings settings) {
         HashSet<ProfileSettings> profiles = new HashSet<>();
+        // Process remote cluster port first so that errors are consistently reported if there
+        // is direct usage of the _remote_cluster profile
+        if (REMOTE_CLUSTER_PORT_ENABLED.get(settings)) {
+            profiles.add(RemoteClusterPortSettings.buildRemoteAccessProfileSettings(settings));
+        }
         boolean isDefaultSet = false;
         for (String profile : settings.getGroups("transport.profiles.", true).keySet()) {
             profiles.add(new ProfileSettings(settings, profile));
@@ -1028,9 +1033,6 @@ public static Set<ProfileSettings> getProfileSettings(Settings settings) {
         if (isDefaultSet == false) {
             profiles.add(new ProfileSettings(settings, TransportSettings.DEFAULT_PROFILE));
         }
-        if (REMOTE_CLUSTER_PORT_ENABLED.get(settings)) {
-            profiles.add(RemoteClusterPortSettings.buildRemoteAccessProfileSettings(settings));
-        }
         // Add the remote access profile
         return Collections.unmodifiableSet(profiles);
     }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java

@@ -100,6 +100,13 @@ private XPackSettings() {
         Setting.Property.NodeScope
     );
 
+    /** Setting for enabling or disabling remote cluster access TLS. Defaults to true. */
+    public static final Setting<Boolean> REMOTE_CLUSTER_SSL_ENABLED = Setting.boolSetting(
+        "xpack.security." + RemoteClusterPortSettings.REMOTE_CLUSTER_PREFIX + "ssl.enabled",
+        true,
+        Property.NodeScope
+    );
+
     /** Setting for enabling or disabling the reserved realm. Defaults to true */
     public static final Setting<Boolean> RESERVED_REALM_ENABLED_SETTING = Setting.boolSetting(
         "xpack.security.authc.reserved_realm.enabled",
@@ -237,6 +244,7 @@ public static Setting<String> defaultStoredHashAlgorithmSetting(String key, Func
 
     public static final SslClientAuthenticationMode CLIENT_AUTH_DEFAULT = SslClientAuthenticationMode.REQUIRED;
     public static final SslClientAuthenticationMode HTTP_CLIENT_AUTH_DEFAULT = SslClientAuthenticationMode.NONE;
+    public static final SslClientAuthenticationMode REMOTE_CLUSTER_CLIENT_AUTH_DEFAULT = SslClientAuthenticationMode.NONE;
     public static final SslVerificationMode VERIFICATION_MODE_DEFAULT = SslVerificationMode.FULL;
 
     // http specific settings
@@ -270,6 +278,9 @@ public static List<Setting<?>> getAllSettings() {
         settings.add(DLS_FLS_ENABLED);
         settings.add(TRANSPORT_SSL_ENABLED);
         settings.add(HTTP_SSL_ENABLED);
+        if (TcpTransport.isUntrustedRemoteClusterEnabled()) {
+            settings.add(REMOTE_CLUSTER_SSL_ENABLED);
+        }
         settings.add(RESERVED_REALM_ENABLED_SETTING);
         settings.add(TOKEN_SERVICE_ENABLED_SETTING);
         settings.add(API_KEY_SERVICE_ENABLED_SETTING);

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SSLService.java

@@ -30,7 +30,6 @@
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.env.Environment;
-import org.elasticsearch.transport.TcpTransport;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.common.socket.SocketAccess;
 import org.elasticsearch.xpack.core.ssl.cert.CertificateInfo;
@@ -79,7 +78,9 @@
 import javax.net.ssl.X509ExtendedTrustManager;
 import javax.security.auth.x500.X500Principal;
 
+import static org.elasticsearch.transport.RemoteClusterPortSettings.REMOTE_CLUSTER_PORT_ENABLED;
 import static org.elasticsearch.xpack.core.XPackSettings.DEFAULT_SUPPORTED_PROTOCOLS;
+import static org.elasticsearch.xpack.core.XPackSettings.REMOTE_CLUSTER_SSL_ENABLED;
 
 /**
  * Provides access to {@link SSLEngine} and {@link SSLSocketFactory} objects based on a provided configuration. All
@@ -591,8 +592,8 @@ static Map<String, Settings> getSSLSettingsMap(Settings settings) {
         sslSettingsMap.put(WatcherField.EMAIL_NOTIFICATION_SSL_PREFIX, settings.getByPrefix(WatcherField.EMAIL_NOTIFICATION_SSL_PREFIX));
         sslSettingsMap.put(XPackSettings.TRANSPORT_SSL_PREFIX, settings.getByPrefix(XPackSettings.TRANSPORT_SSL_PREFIX));
         sslSettingsMap.putAll(getTransportProfileSSLSettings(settings));
-        if (TcpTransport.isUntrustedRemoteClusterEnabled()) {
-            sslSettingsMap.put(XPackSettings.REMOTE_CLUSTER_SSL_PREFIX, settings.getByPrefix(XPackSettings.REMOTE_CLUSTER_SSL_PREFIX));
+        if (REMOTE_CLUSTER_PORT_ENABLED.get(settings)) {
+            sslSettingsMap.put(XPackSettings.REMOTE_CLUSTER_SSL_PREFIX, getRemoteClusterSslSettings(settings));
         }
         return Collections.unmodifiableMap(sslSettingsMap);
     }
@@ -615,6 +616,7 @@ Map<SslConfiguration, SSLContextHolder> loadSslConfigurations(Map<String, SslCon
         for (String context : List.of("xpack.security.transport.ssl", "xpack.security.http.ssl")) {
             validateServerConfiguration(context);
         }
+        maybeValidateRemoteClusterServerConfiguration();
 
         return Collections.unmodifiableMap(sslContextHolders);
     }
@@ -627,18 +629,7 @@ private void validateServerConfiguration(String prefix) {
             // Client Authentication _should_ be required, but if someone turns it off, then this check is no longer relevant
             final SSLConfigurationSettings configurationSettings = SSLConfigurationSettings.withPrefix(prefix + ".", true);
             if (isConfigurationValidForServerUsage(configuration) == false) {
-                throw new ElasticsearchSecurityException(
-                    "invalid SSL configuration for "
-                        + prefix
-                        + " - server ssl configuration requires a key and certificate, but these have not been configured; "
-                        + "you must set either ["
-                        + configurationSettings.x509KeyPair.keystorePath.getKey()
-                        + "], or both ["
-                        + configurationSettings.x509KeyPair.keyPath.getKey()
-                        + "] and ["
-                        + configurationSettings.x509KeyPair.certificatePath.getKey()
-                        + "]"
-                );
+                throwExceptionForMissingKeyMaterial(prefix, configurationSettings);
             }
         } else if (settings.hasValue(enabledSetting) == false) {
             final List<String> sslSettingNames = settings.keySet().stream().filter(s -> s.startsWith(prefix)).sorted().toList();
@@ -656,6 +647,35 @@ private void validateServerConfiguration(String prefix) {
         }
     }
 
+    private void maybeValidateRemoteClusterServerConfiguration() {
+        if (REMOTE_CLUSTER_PORT_ENABLED.get(settings) == false) {
+            return;
+        }
+        final String prefix = "xpack.security.remote_cluster.ssl";
+        final SslConfiguration sslConfiguration = getSSLConfiguration(prefix);
+        if (REMOTE_CLUSTER_SSL_ENABLED.get(settings)) {
+            if (isConfigurationValidForServerUsage(sslConfiguration) == false) {
+                final SSLConfigurationSettings configurationSettings = SSLConfigurationSettings.withPrefix(prefix + ".", false);
+                throwExceptionForMissingKeyMaterial(prefix, configurationSettings);
+            }
+        }
+    }
+
+    private static void throwExceptionForMissingKeyMaterial(String prefix, SSLConfigurationSettings configurationSettings) {
+        throw new ElasticsearchSecurityException(
+            "invalid SSL configuration for "
+                + prefix
+                + " - server ssl configuration requires a key and certificate, but these have not been configured; "
+                + "you must set either ["
+                + configurationSettings.x509KeyPair.keystorePath.getKey()
+                + "], or both ["
+                + configurationSettings.x509KeyPair.keyPath.getKey()
+                + "] and ["
+                + configurationSettings.x509KeyPair.certificatePath.getKey()
+                + "]"
+        );
+    }
+
     /**
      * Returns information about each certificate that is referenced by any SSL configuration.
      * This includes certificates used for identity (with a private key) and those used for trust, but excludes
@@ -881,6 +901,15 @@ private static Settings getHttpTransportSSLSettings(Settings settings) {
         return builder.build();
     }
 
+    private static Settings getRemoteClusterSslSettings(Settings settings) {
+        final Settings remoteClusterSslSettings = settings.getByPrefix(XPackSettings.REMOTE_CLUSTER_SSL_PREFIX);
+        final Settings.Builder builder = Settings.builder().put(remoteClusterSslSettings);
+        if (builder.get("client_authentication") == null) {
+            builder.put("client_authentication", XPackSettings.REMOTE_CLUSTER_CLIENT_AUTH_DEFAULT);
+        }
+        return builder.build();
+    }
+
     public SslConfiguration getHttpTransportSSLConfiguration() {
         return getSSLConfiguration(XPackSettings.HTTP_SSL_PREFIX);
     }

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SslSettingsLoaderTests.java

@@ -15,6 +15,7 @@
 import org.elasticsearch.common.ssl.KeyStoreUtil;
 import org.elasticsearch.common.ssl.PemKeyConfig;
 import org.elasticsearch.common.ssl.PemTrustConfig;
+import org.elasticsearch.common.ssl.SslClientAuthenticationMode;
 import org.elasticsearch.common.ssl.SslConfiguration;
 import org.elasticsearch.common.ssl.SslConfigurationKeys;
 import org.elasticsearch.common.ssl.SslKeyConfig;
@@ -88,6 +89,7 @@ public void testRemoteClusterPortConfigurationIsInjectedWithDefaultsIfEnabled()
         assertThat(sslConfiguration.trustConfig().getClass().getSimpleName(), is("DefaultJdkTrustConfig"));
         assertThat(sslConfiguration.supportedProtocols(), equalTo(XPackSettings.DEFAULT_SUPPORTED_PROTOCOLS));
         assertThat(sslConfiguration.supportedProtocols(), not(hasItem("TLSv1")));
+        assertThat(sslConfiguration.clientAuth(), is(SslClientAuthenticationMode.NONE));
     }
 
     /**
@@ -104,12 +106,14 @@ public void testRemoteClusterPortConfigurationIsInjectedWithItsSettingsIfEnabled
             .put(RemoteClusterPortSettings.REMOTE_CLUSTER_PORT_ENABLED.getKey(), true)
             .put(XPackSettings.REMOTE_CLUSTER_SSL_PREFIX + SslConfigurationKeys.KEYSTORE_PATH, path)
             .putList(XPackSettings.REMOTE_CLUSTER_SSL_PREFIX + SslConfigurationKeys.PROTOCOLS, "TLSv1.3", "TLSv1.2")
+            .put(XPackSettings.REMOTE_CLUSTER_SSL_PREFIX + SslConfigurationKeys.CLIENT_AUTH, "required")
             .setSecureSettings(secureSettings)
             .build();
         Map<String, Settings> settingsMap = SSLService.getSSLSettingsMap(testSettings);
         assertThat(settingsMap, hasKey(XPackSettings.REMOTE_CLUSTER_SSL_PREFIX));
         SslConfiguration sslConfiguration = getSslConfiguration(settingsMap.get(XPackSettings.REMOTE_CLUSTER_SSL_PREFIX));
         assertThat(sslConfiguration.supportedProtocols(), contains("TLSv1.3", "TLSv1.2"));
+        assertThat(sslConfiguration.clientAuth(), is(SslClientAuthenticationMode.REQUIRED));
 
         SslKeyConfig keyStore = sslConfiguration.keyConfig();
         assertThat(keyStore.getDependentFiles(), contains(path));

x-pack/plugin/security/qa/multi-cluster/build.gradle

@@ -6,76 +6,88 @@
  */
 
 import org.elasticsearch.gradle.Version
+import org.elasticsearch.gradle.internal.info.BuildParams
 import org.elasticsearch.gradle.internal.test.RestIntegTestTask
 
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-resources'
 
-configurations {
-  signedCerts
-  rootCert
-}
-
-dependencies {
-  signedCerts project(path: ':x-pack:plugin:core', configuration: 'signedCerts')
-  rootCert project(path: ':x-pack:plugin:core', configuration: 'rootCert')
-}
-
-tasks.register("copyCerts", Sync) {
-  dependsOn configurations.signedCerts
-  from(configurations.signedCerts)
-  from(configurations.rootCert)
-  into "${buildDir}/certs"
-}
-
-
 def fulfillingCluster = testClusters.register('fulfilling-cluster') {
   requiresFeature 'es.untrusted_remote_cluster_feature_flag_registered', Version.fromString("8.5.0")
+  setting 'xpack.security.enabled', 'true'
   setting 'xpack.license.self_generated.type', 'trial'
   setting 'remote_cluster.enabled', 'true'
   setting 'remote_cluster.port', '0'
 
-  extraConfigFile 'transport.key', file("${buildDir}/certs/n1.c1.key")
-  extraConfigFile 'transport.cert', file("${buildDir}/certs/n1.c1.crt")
-  extraConfigFile 'transport.ca', file("${buildDir}/certs/ca.crt")
+  extraConfigFile 'transport.key', file("src/test/resources/ssl/transport.key")
+  extraConfigFile 'transport.cert', file("src/test/resources/ssl/transport.crt")
+  extraConfigFile 'transport.ca', file("src/test/resources/ssl/transport-ca.crt")
 
-  setting 'xpack.security.transport.ssl.enabled', 'true'
+  // Transport SSL can be enabled or disabled. It is independent from the remote cluster server SSL
+  if (BuildParams.random.nextBoolean()) {
+    setting 'xpack.security.transport.ssl.enabled', 'true'
+  } else {
+    setting 'xpack.security.transport.ssl.enabled', 'false'
+  }
   setting 'xpack.security.transport.ssl.key', 'transport.key'
   setting 'xpack.security.transport.ssl.certificate', 'transport.cert'
+  setting 'xpack.security.transport.ssl.key_passphrase', 'transport-password'
   setting 'xpack.security.transport.ssl.certificate_authorities', 'transport.ca'
   setting 'xpack.security.transport.ssl.client_authentication', 'required'
   setting 'xpack.security.transport.ssl.verification_mode', 'certificate'
 
-  // It is intentionally to use none for both verification_mode and client_authentication.
-  // Because SSL is not wired up properly on the client (QC) side. These settings
-  // just test that they can be configured.
-  // Once SSL is all wired up, we will need (1) proper SSL verification and
-  // (2) different set of key and cert than the ones used for the transport interface.
-  setting 'xpack.security.remote_cluster.ssl.key', 'transport.key'
-  setting 'xpack.security.remote_cluster.ssl.certificate', 'transport.cert'
-  setting 'xpack.security.remote_cluster.ssl.verification_mode', 'none'
-  setting 'xpack.security.remote_cluster.ssl.client_authentication', 'none'
+  // Server side SSL configuration for remote cluster
+  extraConfigFile 'remote-cluster.key', file("src/test/resources/ssl/remote_cluster.key")
+  extraConfigFile 'remote-cluster.cert', file("src/test/resources/ssl/remote_cluster.crt")
+  extraConfigFile 'remote-cluster.ca', file("src/test/resources/ssl/remote-cluster-ca.crt")
+
+  setting 'xpack.security.remote_cluster.ssl.enabled', 'true'
+  setting 'xpack.security.remote_cluster.ssl.key', 'remote-cluster.key'
+  setting 'xpack.security.remote_cluster.ssl.certificate', 'remote-cluster.cert'
+  keystore 'xpack.security.remote_cluster.ssl.secure_key_passphrase', 'remote-cluster-password'
+  // client auth defaults to none for remote_cluster
+  if (BuildParams.random.nextBoolean()) {
+    setting 'xpack.security.remote_cluster.ssl.client_authentication', 'none'
+  }
+
+  user username: "test_user", password: "x-pack-test-password"
 }
 
 def queryingCluster = testClusters.register('querying-cluster') {
   requiresFeature 'es.untrusted_remote_cluster_feature_flag_registered', Version.fromString("8.5.0")
+  setting 'xpack.security.enabled', 'true'
   setting 'xpack.license.self_generated.type', 'trial'
   setting 'cluster.remote.connections_per_cluster', "1"
 
+  // TODO: For now, the client SSL configuration on the query cluster side is shared between
+  //       the default transport and remote_cluster profiles.
+  //       Therefore we cannot configure them separately. This will be separated once we add the support.
+  extraConfigFile 'transport.key', file("src/test/resources/ssl/remote_cluster.key")
+  extraConfigFile 'transport.cert', file("src/test/resources/ssl/remote_cluster.crt")
+  extraConfigFile 'transport.ca', file("src/test/resources/ssl/remote-cluster-ca.crt")
+
+  setting 'xpack.security.transport.ssl.enabled', 'true'
+  setting 'xpack.security.transport.ssl.key', 'transport.key'
+  setting 'xpack.security.transport.ssl.certificate', 'transport.cert'
+  setting 'xpack.security.transport.ssl.key_passphrase', 'remote-cluster-password'
+  setting 'xpack.security.transport.ssl.certificate_authorities', 'transport.ca'
+  setting 'xpack.security.transport.ssl.client_authentication', 'required'
+  setting 'xpack.security.transport.ssl.verification_mode', 'certificate'
+
   setting 'cluster.remote.my_remote_cluster.mode', 'proxy'
   setting 'cluster.remote.my_remote_cluster.proxy_address', {
-    "\"${fulfillingCluster.get().getAllRemoteAccessPortURI() .get(0)}\""
+    "\"${fulfillingCluster.get().getAllRemoteAccessPortURI().get(0)}\""
   }
+
+  user username: "test_user", password: "x-pack-test-password"
 }
 
 tasks.register('fulfilling-cluster', RestIntegTestTask) {
-  dependsOn 'copyCerts'
   useCluster fulfillingCluster
   systemProperty 'tests.rest.suite', 'fulfilling_cluster'
 }
 
 tasks.register('querying-cluster', RestIntegTestTask) {
-  dependsOn 'copyCerts'
   dependsOn 'fulfilling-cluster'
   useCluster queryingCluster
   useCluster fulfillingCluster
@@ -84,7 +96,6 @@ tasks.register('querying-cluster', RestIntegTestTask) {
 
 // runs the fulfilling-cluster cluster tests then the querying-cluster tests
 tasks.register("integTest") {
-  dependsOn 'copyCerts'
   dependsOn 'querying-cluster'
 }
 

x-pack/plugin/security/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecuritySmokeIT.java

@@ -9,6 +9,9 @@
 
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.test.rest.ObjectPath;
 
@@ -29,6 +32,12 @@ protected boolean preserveDataStreamsUponCompletion() {
         return true;
     }
 
+    @Override
+    protected Settings restClientSettings() {
+        String token = basicAuthHeaderValue("test_user", new SecureString("x-pack-test-password".toCharArray()));
+        return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build();
+    }
+
     private boolean isFulfillingCluster() {
         return "fulfilling_cluster".equals(System.getProperty("tests.rest.suite"));
     }