8371721: Refactor checkTrusted methods in X509TrustManagerImpl

Reviewed-by: coffeys, djelinski

Author: artur-oracle

src/java.base/share/classes/sun/security/ssl/X509TrustManagerImpl.java

@@ -30,29 +30,22 @@
 import java.security.cert.*;
 import java.util.*;
 import java.util.concurrent.locks.ReentrantLock;
-
 import javax.net.ssl.*;
-
 import sun.security.ssl.SSLAlgorithmConstraints.SIGNATURE_CONSTRAINTS_MODE;
 import sun.security.util.AnchorCertificates;
 import sun.security.util.HostnameChecker;
 import sun.security.validator.*;
 
 /**
  * This class implements the SunJSSE X.509 trust manager using the internal
- * validator API in J2SE core. The logic in this class is minimal.<p>
+ * validator API in J2SE core. The logic in this class is minimal.
  * <p>
  * This class supports both the Simple validation algorithm from previous
- * JSSE versions and PKIX validation. Currently, it is not possible for the
- * application to specify PKIX parameters other than trust anchors. This will
- * be fixed in a future release using new APIs. When that happens, it may also
- * make sense to separate the Simple and PKIX trust managers into separate
- * classes.
+ * JSSE versions and PKIX validation.
  *
  * @author Andreas Sterbenz
  */
-final class X509TrustManagerImpl extends X509ExtendedTrustManager
-        implements X509TrustManager {
+final class X509TrustManagerImpl extends X509ExtendedTrustManager {
 
     private final String validatorType;
 
@@ -208,120 +201,90 @@ private Validator checkTrustedInit(X509Certificate[] chain,
     private void checkTrusted(X509Certificate[] chain,
             String authType, Socket socket,
             boolean checkClientTrusted) throws CertificateException {
-        Validator v = checkTrustedInit(chain, authType, checkClientTrusted);
 
-        X509Certificate[] trustedChain;
+        SSLSession session = null;
+        AlgorithmConstraints constraints = null;
+        String identityAlg = null;
+
         if (socket instanceof SSLSocket sslSocket && sslSocket.isConnected()) {
+            session = sslSocket.getHandshakeSession();
+            constraints = SSLAlgorithmConstraints.forSocket(
+                    sslSocket, SIGNATURE_CONSTRAINTS_MODE.LOCAL, false);
+            identityAlg = sslSocket.getSSLParameters().
+                    getEndpointIdentificationAlgorithm();
+        }
 
-            SSLSession session = sslSocket.getHandshakeSession();
-            if (session == null) {
-                throw new CertificateException("No handshake session");
-            }
+        findTrustedCertificate(chain, authType,
+                session, constraints, identityAlg, checkClientTrusted);
+    }
 
-            AlgorithmConstraints constraints = SSLAlgorithmConstraints.forSocket(
-                    sslSocket, SIGNATURE_CONSTRAINTS_MODE.LOCAL, false);
+    private void checkTrusted(X509Certificate[] chain,
+            String authType, SSLEngine engine,
+            boolean checkClientTrusted) throws CertificateException {
 
-            // Grab any stapled OCSP responses for use in validation
-            List<byte[]> responseList = Collections.emptyList();
-            if (!checkClientTrusted && session instanceof ExtendedSSLSession) {
-                responseList =
-                        ((ExtendedSSLSession)session).getStatusResponses();
-            }
-            trustedChain = v.validate(chain, null, responseList,
-                    constraints, checkClientTrusted ? null : authType);
+        SSLSession session = null;
+        AlgorithmConstraints constraints = null;
+        String identityAlg = null;
 
-            // check endpoint identity
-            String identityAlg = sslSocket.getSSLParameters().
+        if (engine != null) {
+            session = engine.getHandshakeSession();
+            constraints = SSLAlgorithmConstraints.forEngine(
+                    engine, SIGNATURE_CONSTRAINTS_MODE.LOCAL, false);
+            identityAlg = engine.getSSLParameters().
                     getEndpointIdentificationAlgorithm();
-            if (identityAlg != null && !identityAlg.isEmpty()) {
-                checkIdentity(session,
-                        trustedChain, identityAlg, checkClientTrusted);
-            }
-        } else {
-            trustedChain = v.validate(chain, null, Collections.emptyList(),
-                    null, checkClientTrusted ? null : authType);
         }
 
-        if (SSLLogger.isOn() && SSLLogger.isOn("ssl,trustmanager")) {
-            SSLLogger.fine("Found trusted certificate",
-                    trustedChain[trustedChain.length - 1]);
-        }
+        findTrustedCertificate(chain, authType,
+                session, constraints, identityAlg, checkClientTrusted);
     }
 
     private void checkTrusted(X509Certificate[] chain,
             String authType, QuicTLSEngineImpl quicTLSEngine,
             boolean checkClientTrusted) throws CertificateException {
-        Validator v = checkTrustedInit(chain, authType, checkClientTrusted);
 
-        final X509Certificate[] trustedChain;
-        if (quicTLSEngine != null) {
+        SSLSession session = null;
+        AlgorithmConstraints constraints = null;
+        String identityAlg = null;
 
-            final SSLSession session = quicTLSEngine.getHandshakeSession();
-            if (session == null) {
-                throw new CertificateException("No handshake session");
-            }
-
-            // create the algorithm constraints
-            final AlgorithmConstraints constraints = SSLAlgorithmConstraints.forQUIC(
+        if (quicTLSEngine != null) {
+            session = quicTLSEngine.getHandshakeSession();
+            constraints = SSLAlgorithmConstraints.forQUIC(
                     quicTLSEngine, SIGNATURE_CONSTRAINTS_MODE.LOCAL, false);
-            final List<byte[]> responseList;
-            // grab any stapled OCSP responses for use in validation
-            if (!checkClientTrusted &&
-                    session instanceof ExtendedSSLSession extSession) {
-                responseList = extSession.getStatusResponses();
-            } else {
-                responseList = Collections.emptyList();
-            }
-            // do the certificate chain validation
-            trustedChain = v.validate(chain, null, responseList,
-                    constraints, checkClientTrusted ? null : authType);
-
-            // check endpoint identity
-            String identityAlg = quicTLSEngine.getSSLParameters().
+            identityAlg = quicTLSEngine.getSSLParameters().
                     getEndpointIdentificationAlgorithm();
-            if (identityAlg != null && !identityAlg.isEmpty()) {
-                checkIdentity(session, trustedChain,
-                        identityAlg, checkClientTrusted);
-            }
-        } else {
-            trustedChain = v.validate(chain, null, Collections.emptyList(),
-                    null, checkClientTrusted ? null : authType);
         }
 
-        if (SSLLogger.isOn() && SSLLogger.isOn("ssl,trustmanager")) {
-            SSLLogger.fine("Found trusted certificate",
-                    trustedChain[trustedChain.length - 1]);
-        }
+        findTrustedCertificate(chain, authType,
+                session, constraints, identityAlg, checkClientTrusted);
     }
 
-    private void checkTrusted(X509Certificate[] chain,
-            String authType, SSLEngine engine,
+    private void findTrustedCertificate(X509Certificate[] chain,
+            String authType, SSLSession session,
+            AlgorithmConstraints constraints, String identityAlg,
             boolean checkClientTrusted) throws CertificateException {
-        Validator v = checkTrustedInit(chain, authType, checkClientTrusted);
 
-        X509Certificate[] trustedChain;
-        if (engine != null) {
+        final X509Certificate[] trustedChain;
+        Validator v = checkTrustedInit(chain, authType, checkClientTrusted);
 
-            SSLSession session = engine.getHandshakeSession();
+        if (constraints != null) {
             if (session == null) {
                 throw new CertificateException("No handshake session");
             }
 
-            AlgorithmConstraints constraints = SSLAlgorithmConstraints.forEngine(
-                    engine, SIGNATURE_CONSTRAINTS_MODE.LOCAL, false);
-
             // Grab any stapled OCSP responses for use in validation
-            List<byte[]> responseList = Collections.emptyList();
-            if (!checkClientTrusted && session instanceof ExtendedSSLSession) {
-                responseList =
-                        ((ExtendedSSLSession)session).getStatusResponses();
+            final List<byte[]> responseList;
+            if (!checkClientTrusted &&
+                    session instanceof ExtendedSSLSession extSession) {
+                responseList = extSession.getStatusResponses();
+            } else {
+                responseList = Collections.emptyList();
             }
+
+            // Certificate chain validation
             trustedChain = v.validate(chain, null, responseList,
                     constraints, checkClientTrusted ? null : authType);
 
-            // check endpoint identity
-            String identityAlg = engine.getSSLParameters().
-                    getEndpointIdentificationAlgorithm();
+            // Check endpoint identity
             if (identityAlg != null && !identityAlg.isEmpty()) {
                 checkIdentity(session, trustedChain,
                         identityAlg, checkClientTrusted);