DSP-24887: Enable SAN-aware peer identity lookup for Endpoint verifications on hostname

.github/workflows/build-and-publish.yml

@@ -7,7 +7,7 @@ on:
   # Trigger on version tags
   push:
     branches:
-      - dse-netty-4.1.135
+      - dse-netty-4.1.135.2
     tags:
       - '*.dse'
       - 'dse-netty-*'

handler/src/main/java/io/netty/handler/ssl/JdkSslClientContext.java

@@ -176,8 +176,8 @@ public JdkSslClientContext(
                         long sessionCacheSize, long sessionTimeout) throws SSLException {
         super(newSSLContext(provider, toX509CertificatesInternal(trustCertCollectionFile),
                 trustManagerFactory, null, null,
-                null, null, sessionCacheSize, sessionTimeout, null, KeyStore.getDefaultType(), null), true,
-                ciphers, cipherFilter, apn, ClientAuth.NONE, null, false);
+                null, null, sessionCacheSize, sessionTimeout, null, KeyStore.getDefaultType(),
+                null, false), true, ciphers, cipherFilter, apn, ClientAuth.NONE, null, false);
     }
 
     /**
@@ -260,7 +260,7 @@ public JdkSslClientContext(File trustCertCollectionFile, TrustManagerFactory tru
                 trustCertCollectionFile), trustManagerFactory,
                 toX509CertificatesInternal(keyCertChainFile), toPrivateKeyInternal(keyFile, keyPassword),
                 keyPassword, keyManagerFactory, sessionCacheSize, sessionTimeout,
-                        null, KeyStore.getDefaultType(), null), true,
+                null, KeyStore.getDefaultType(), null, false), true,
                 ciphers, cipherFilter, apn, ClientAuth.NONE, null, false);
     }
 
@@ -270,11 +270,11 @@ public JdkSslClientContext(File trustCertCollectionFile, TrustManagerFactory tru
                         KeyManagerFactory keyManagerFactory, Iterable<String> ciphers, CipherSuiteFilter cipherFilter,
                         ApplicationProtocolConfig apn, String[] protocols, long sessionCacheSize, long sessionTimeout,
                         SecureRandom secureRandom, String keyStoreType, String endpointIdentificationAlgorithm,
-                        ResumptionController resumptionController)
+                        ResumptionController resumptionController, boolean sanPeerIdentityLookup)
             throws SSLException {
         super(newSSLContext(sslContextProvider, trustCertCollection, trustManagerFactory,
                             keyCertChain, key, keyPassword, keyManagerFactory, sessionCacheSize,
-                            sessionTimeout, secureRandom, keyStoreType, resumptionController),
+                            sessionTimeout, secureRandom, keyStoreType, resumptionController, sanPeerIdentityLookup),
                 true, ciphers, cipherFilter, toNegotiator(apn, false), ClientAuth.NONE, protocols, false,
                 endpointIdentificationAlgorithm, resumptionController);
     }
@@ -285,7 +285,8 @@ private static SSLContext newSSLContext(Provider sslContextProvider,
                                             PrivateKey key, String keyPassword, KeyManagerFactory keyManagerFactory,
                                             long sessionCacheSize, long sessionTimeout,
                                             SecureRandom secureRandom, String keyStore,
-                                            ResumptionController resumptionController) throws SSLException {
+                                            ResumptionController resumptionController,
+                                            boolean sanPeerIdentityLookup) throws SSLException {
         try {
             if (trustCertCollection != null) {
                 trustManagerFactory = buildTrustManagerFactory(trustCertCollection, trustManagerFactory, keyStore);
@@ -297,8 +298,8 @@ private static SSLContext newSSLContext(Provider sslContextProvider,
             SSLContext ctx = sslContextProvider == null ? SSLContext.getInstance(PROTOCOL)
                 : SSLContext.getInstance(PROTOCOL, sslContextProvider);
             ctx.init(keyManagerFactory == null ? null : keyManagerFactory.getKeyManagers(),
-                     trustManagerFactory == null ? null :
-                             wrapIfNeeded(trustManagerFactory.getTrustManagers(), resumptionController),
+                     trustManagerFactory == null ? null : wrapIfNeeded(
+                             trustManagerFactory.getTrustManagers(), resumptionController, sanPeerIdentityLookup),
                      secureRandom);
 
             SSLSessionContext sessCtx = ctx.getClientSessionContext();
@@ -317,12 +318,16 @@ private static SSLContext newSSLContext(Provider sslContextProvider,
         }
     }
 
-    private static TrustManager[] wrapIfNeeded(TrustManager[] tms, ResumptionController resumptionController) {
-        if (tms == null || resumptionController == null) {
+    private static TrustManager[] wrapIfNeeded(TrustManager[] tms, ResumptionController resumptionController,
+                                               boolean sanPeerIdentityLookup) {
+        if (tms == null) {
             return tms;
         }
         for (int i = 0; i < tms.length; i++) {
-            tms[i] = resumptionController.wrapIfNeeded(tms[i]);
+            tms[i] = SanPeerIdentityTrustManager.wrapIfNeeded(tms[i], sanPeerIdentityLookup);
+            if (resumptionController != null) {
+                tms[i] = resumptionController.wrapIfNeeded(tms[i]);
+            }
         }
         return tms;
     }

handler/src/main/java/io/netty/handler/ssl/OpenSslClientContext.java

@@ -196,7 +196,8 @@ public OpenSslClientContext(File trustCertCollectionFile, TrustManagerFactory tr
             OpenSslKeyMaterialProvider.validateKeyMaterialSupported(keyCertChain, key, keyPassword);
             sessionContext = newSessionContext(this, ctx, engineMap, trustCertCollection, trustManagerFactory,
                                                keyCertChain, key, keyPassword, keyManagerFactory, keyStore,
-                                               sessionCacheSize, sessionTimeout, resumptionController);
+                                               sessionCacheSize, sessionTimeout, endpointIdentificationAlgorithm,
+                                               resumptionController, options);
             success = true;
         } finally {
             if (!success) {

handler/src/main/java/io/netty/handler/ssl/ReferenceCountedOpenSslClientContext.java

@@ -73,7 +73,8 @@ public final class ReferenceCountedOpenSslClientContext extends ReferenceCounted
         try {
             sessionContext = newSessionContext(this, ctx, engineMap, trustCertCollection, trustManagerFactory,
                                                keyCertChain, key, keyPassword, keyManagerFactory, keyStore,
-                                               sessionCacheSize, sessionTimeout, resumptionController);
+                                               sessionCacheSize, sessionTimeout, endpointIdentificationAlgorithm,
+                                               resumptionController, options);
             success = true;
         } finally {
             if (!success) {
@@ -94,7 +95,9 @@ static OpenSslSessionContext newSessionContext(ReferenceCountedOpenSslContext th
                                                    X509Certificate[] keyCertChain, PrivateKey key,
                                                    String keyPassword, KeyManagerFactory keyManagerFactory,
                                                    String keyStore, long sessionCacheSize, long sessionTimeout,
-                                                   ResumptionController resumptionController)
+                                                   String endpointIdentificationAlgorithm,
+                                                   ResumptionController resumptionController,
+                                                   Map.Entry<SslContextOption<?>, Object>... options)
             throws SSLException {
         if (key == null && keyCertChain != null || key != null && keyCertChain == null) {
             throw new IllegalArgumentException(
@@ -155,8 +158,11 @@ static OpenSslSessionContext newSessionContext(ReferenceCountedOpenSslContext th
                             TrustManagerFactory.getDefaultAlgorithm());
                     trustManagerFactory.init((KeyStore) null);
                 }
-                final X509TrustManager manager = chooseTrustManager(
-                        trustManagerFactory.getTrustManagers(), resumptionController);
+                final boolean sanPeerIdentityLookup = isSanPeerIdentityLookupEnabled(
+                        endpointIdentificationAlgorithm, options);
+                final X509TrustManager manager = wrapTrustManagerIfNeeded(
+                        chooseTrustManager(trustManagerFactory.getTrustManagers(), resumptionController),
+                        sanPeerIdentityLookup);
 
                 // IMPORTANT: The callbacks set for verification must be static to prevent memory leak as
                 //            otherwise the context can never be collected. This is because the JNI code holds
@@ -193,6 +199,37 @@ static OpenSslSessionContext newSessionContext(ReferenceCountedOpenSslContext th
         }
     }
 
+    private static boolean isSanPeerIdentityLookupEnabled(String endpointIdentificationAlgorithm,
+                                                          Map.Entry<SslContextOption<?>, Object>[] options) {
+        if (endpointIdentificationAlgorithm == null) {
+            return false;
+        }
+        if (options == null) {
+            return false;
+        }
+        for (Map.Entry<SslContextOption<?>, Object> option : options) {
+            if (option == null) {
+                continue;
+            }
+            if (SanPeerIdentityConfig.SAN_PEER_IDENTITY_LOOKUP.equals(option.getKey())) {
+                return Boolean.TRUE.equals(option.getValue());
+            }
+        }
+        return false;
+    }
+
+    @SuppressJava6Requirement(reason = "Usage guarded by java version check")
+    private static X509TrustManager wrapTrustManagerIfNeeded(X509TrustManager manager,
+                                                             boolean sanPeerIdentityLookup) {
+        if (!sanPeerIdentityLookup) {
+            return manager;
+        }
+        if (useExtendedTrustManager(manager)) {
+            return new SanPeerIdentityTrustManager((X509ExtendedTrustManager) manager);
+        }
+        return manager;
+    }
+
     @SuppressJava6Requirement(reason = "Guarded by java version check")
     private static void setVerifyCallback(long ctx, OpenSslEngineMap engineMap, X509TrustManager manager) {
         // Use this to prevent an error when running on java < 7

handler/src/main/java/io/netty/handler/ssl/SanPeerIdentityConfig.java

@@ -0,0 +1,24 @@
+/*
+ * Copyright 2026 The Netty Project
+ *
+ * The Netty Project licenses this file to you under the Apache License,
+ * version 2.0 (the "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at:
+ *
+ *   https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package io.netty.handler.ssl;
+
+final class SanPeerIdentityConfig {
+    static final SslContextOption<Boolean> SAN_PEER_IDENTITY_LOOKUP =
+            new SslContextOption<Boolean>("SAN_PEER_IDENTITY_LOOKUP");
+
+    private SanPeerIdentityConfig() {
+    }
+}