Enhance CertificateException message when throw due hostname validation

handler/src/main/java/io/netty/handler/ssl/EnhancingX509ExtendedTrustManager.java

@@ -0,0 +1,124 @@
+/*
+ * Copyright 2023 The Netty Project
+ *
+ * The Netty Project licenses this file to you under the Apache License,
+ * version 2.0 (the "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at:
+ *
+ *   https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.netty.handler.ssl;
+
+import io.netty.util.internal.SuppressJava6Requirement;
+
+import javax.net.ssl.SSLEngine;
+import javax.net.ssl.X509ExtendedTrustManager;
+import javax.net.ssl.X509TrustManager;
+import java.net.Socket;
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
+import java.util.Collection;
+import java.util.List;
+
+
+/**
+ * Wraps an existing {@link X509ExtendedTrustManager} and enhances the {@link CertificateException} that is thrown
+ * because of hostname validation.
+ */
+@SuppressJava6Requirement(reason = "Usage guarded by java version check")
+final class EnhancingX509ExtendedTrustManager extends X509ExtendedTrustManager {
+    private final X509ExtendedTrustManager wrapped;
+
+    EnhancingX509ExtendedTrustManager(X509TrustManager wrapped) {
+        this.wrapped = (X509ExtendedTrustManager) wrapped;
+    }
+
+    @Override
+    public void checkClientTrusted(X509Certificate[] chain, String authType, Socket socket)
+            throws CertificateException {
+        wrapped.checkClientTrusted(chain, authType, socket);
+    }
+
+    @Override
+    public void checkServerTrusted(X509Certificate[] chain, String authType, Socket socket)
+            throws CertificateException {
+        try {
+            wrapped.checkServerTrusted(chain, authType, socket);
+        } catch (CertificateException e) {
+            throwEnhancedCertificateException(chain, e);
+        }
+    }
+
+    @Override
+    public void checkClientTrusted(X509Certificate[] chain, String authType, SSLEngine engine)
+            throws CertificateException {
+        wrapped.checkClientTrusted(chain, authType, engine);
+    }
+
+    @Override
+    public void checkServerTrusted(X509Certificate[] chain, String authType, SSLEngine engine)
+            throws CertificateException {
+        try {
+            wrapped.checkServerTrusted(chain, authType, engine);
+        } catch (CertificateException e) {
+            throwEnhancedCertificateException(chain, e);
+        }
+    }
+
+    @Override
+    public void checkClientTrusted(X509Certificate[] chain, String authType)
+            throws CertificateException {
+        wrapped.checkClientTrusted(chain, authType);
+    }
+
+    @Override
+    public void checkServerTrusted(X509Certificate[] chain, String authType)
+            throws CertificateException {
+        try {
+            wrapped.checkServerTrusted(chain, authType);
+        } catch (CertificateException e) {
+            throwEnhancedCertificateException(chain, e);
+        }
+    }
+
+    @Override
+    public X509Certificate[] getAcceptedIssuers() {
+        return wrapped.getAcceptedIssuers();
+    }
+
+    private static void throwEnhancedCertificateException(X509Certificate[] chain, CertificateException e)
+            throws CertificateException {
+        // Matching the message is the best we can do sadly.
+        String message = e.getMessage();
+        if (message != null && e.getMessage().startsWith("No subject alternative DNS name matching")) {
+            StringBuilder names = new StringBuilder(64);
+            for (int i = 0; i < chain.length; i++) {
+                X509Certificate cert = chain[i];
+                Collection<List<?>> collection = cert.getSubjectAlternativeNames();
+                if (collection != null) {
+                    for (List<?> altNames : collection) {
+                        // 2 is dNSName. See X509Certificate javadocs.
+                        if (altNames.size() >= 2 && ((Integer) altNames.get(0)).intValue() == 2) {
+                            names.append((String) altNames.get(1)).append(",");
+                        }
+                    }
+                }
+            }
+            if (names.length() != 0) {
+                // Strip of ,
+                names.setLength(names.length() - 1);
+                throw new CertificateException(message +
+                        " Subject alternative DNS names in the certificate chain of " + chain.length +
+                        " certificate(s): " + names, e);
+            }
+        }
+        throw e;
+    }
+}

handler/src/main/java/io/netty/handler/ssl/JdkSslServerContext.java

@@ -16,6 +16,9 @@
 
 package io.netty.handler.ssl;
 
+import io.netty.util.internal.PlatformDependent;
+import io.netty.util.internal.SuppressJava6Requirement;
+
 import java.security.KeyStore;
 import java.security.Provider;
 import javax.net.ssl.KeyManager;
@@ -26,6 +29,7 @@
 import javax.net.ssl.SSLSessionContext;
 import javax.net.ssl.TrustManager;
 import javax.net.ssl.TrustManagerFactory;
+import javax.net.ssl.X509ExtendedTrustManager;
 import java.io.File;
 import java.security.PrivateKey;
 import java.security.cert.X509Certificate;
@@ -261,7 +265,13 @@ private static SSLContext newSSLContext(Provider sslContextProvider, X509Certifi
         try {
             if (trustCertCollection != null) {
                 trustManagerFactory = buildTrustManagerFactory(trustCertCollection, trustManagerFactory, keyStore);
+            } else if (trustManagerFactory == null) {
+                // Mimic the way SSLContext.getInstance(KeyManager[], null, null) works
+                trustManagerFactory = TrustManagerFactory.getInstance(
+                        TrustManagerFactory.getDefaultAlgorithm());
+                trustManagerFactory.init((KeyStore) null);
             }
+
             if (key != null) {
                 keyManagerFactory = buildKeyManagerFactory(keyCertChain, null,
                         key, keyPassword, keyManagerFactory, null);
@@ -271,7 +281,7 @@ private static SSLContext newSSLContext(Provider sslContextProvider, X509Certifi
             SSLContext ctx = sslContextProvider == null ? SSLContext.getInstance(PROTOCOL)
                 : SSLContext.getInstance(PROTOCOL, sslContextProvider);
             ctx.init(keyManagerFactory.getKeyManagers(),
-                     trustManagerFactory == null ? null : trustManagerFactory.getTrustManagers(),
+                    wrapTrustManagerIfNeeded(trustManagerFactory.getTrustManagers()),
                      null);
 
             SSLSessionContext sessCtx = ctx.getServerSessionContext();
@@ -290,4 +300,18 @@ private static SSLContext newSSLContext(Provider sslContextProvider, X509Certifi
         }
     }
 
+    @SuppressJava6Requirement(reason = "Guarded by java version check")
+    private static TrustManager[] wrapTrustManagerIfNeeded(TrustManager[] trustManagers) {
+        if (PlatformDependent.javaVersion() >= 7) {
+            for (int i = 0; i < trustManagers.length; i++) {
+                TrustManager tm = trustManagers[i];
+                if (tm instanceof X509ExtendedTrustManager) {
+                    // Wrap the TrustManager to provide a better exception message for users to debug hostname
+                    // validation failures.
+                    trustManagers[i] = new EnhancingX509ExtendedTrustManager((X509ExtendedTrustManager) tm);
+                }
+            }
+        }
+        return trustManagers;
+    }
 }

handler/src/main/java/io/netty/handler/ssl/ReferenceCountedOpenSslContext.java

@@ -660,10 +660,16 @@ protected static X509Certificate[] certificates(byte[][] chain) {
     protected static X509TrustManager chooseTrustManager(TrustManager[] managers) {
         for (TrustManager m : managers) {
             if (m instanceof X509TrustManager) {
+                X509TrustManager tm = (X509TrustManager) m;
                 if (PlatformDependent.javaVersion() >= 7) {
-                    return OpenSslX509TrustManagerWrapper.wrapIfNeeded((X509TrustManager) m);
+                    tm = OpenSslX509TrustManagerWrapper.wrapIfNeeded((X509TrustManager) m);
+                    if (useExtendedTrustManager(tm)) {
+                        // Wrap the TrustManager to provide a better exception message for users to debug hostname
+                        // validation failures.
+                        tm = new EnhancingX509ExtendedTrustManager(tm);
+                    }
                 }
-                return (X509TrustManager) m;
+                return tm;
             }
         }
         throw new IllegalStateException("no X509TrustManager found");