8271199: Mutual TLS handshake fails signing client certificate with c…

…ustom sensitive PKCS11 key

Reviewed-by: mdoerr
Backport-of: f6232982b91cb2314e96ddbde3984836a810a556

src/java.base/share/classes/sun/security/rsa/RSAPSSSignature.java

@@ -127,12 +127,15 @@ public RSAPSSSignature() {
     @Override
     protected void engineInitVerify(PublicKey publicKey)
             throws InvalidKeyException {
-        if (!(publicKey instanceof RSAPublicKey)) {
+        if (publicKey instanceof RSAPublicKey) {
+            RSAPublicKey rsaPubKey = (RSAPublicKey)publicKey;
+            isPublicKeyValid(rsaPubKey);
+            this.pubKey = rsaPubKey;
+            this.privKey = null;
+            resetDigest();
+        } else {
             throw new InvalidKeyException("key must be RSAPublicKey");
         }
-        this.pubKey = (RSAPublicKey) isValid((RSAKey)publicKey);
-        this.privKey = null;
-        resetDigest();
     }
 
     // initialize for signing. See JCA doc
@@ -146,14 +149,17 @@ protected void engineInitSign(PrivateKey privateKey)
     @Override
     protected void engineInitSign(PrivateKey privateKey, SecureRandom random)
             throws InvalidKeyException {
-        if (!(privateKey instanceof RSAPrivateKey)) {
+        if (privateKey instanceof RSAPrivateKey) {
+            RSAPrivateKey rsaPrivateKey = (RSAPrivateKey)privateKey;
+            isPrivateKeyValid(rsaPrivateKey);
+            this.privKey = rsaPrivateKey;
+            this.pubKey = null;
+            this.random =
+                    (random == null ? JCAUtil.getSecureRandom() : random);
+            resetDigest();
+        } else {
             throw new InvalidKeyException("key must be RSAPrivateKey");
         }
-        this.privKey = (RSAPrivateKey) isValid((RSAKey)privateKey);
-        this.pubKey = null;
-        this.random =
-            (random == null? JCAUtil.getSecureRandom() : random);
-        resetDigest();
     }
 
     /**
@@ -205,11 +211,57 @@ private static boolean isCompatible(AlgorithmParameterSpec keyParams,
         }
     }
 
+    /**
+     * Validate the specified RSAPrivateKey
+     */
+    private void isPrivateKeyValid(RSAPrivateKey prKey)  throws InvalidKeyException {
+        try {
+            if (prKey instanceof RSAPrivateCrtKey) {
+                RSAPrivateCrtKey crtKey = (RSAPrivateCrtKey)prKey;
+                if (RSAPrivateCrtKeyImpl.checkComponents(crtKey)) {
+                    RSAKeyFactory.checkRSAProviderKeyLengths(
+                            crtKey.getModulus().bitLength(),
+                            crtKey.getPublicExponent());
+                } else {
+                    throw new InvalidKeyException(
+                            "Some of the CRT-specific components are not available");
+                }
+            } else {
+                RSAKeyFactory.checkRSAProviderKeyLengths(
+                        prKey.getModulus().bitLength(),
+                        null);
+            }
+        } catch (InvalidKeyException ikEx) {
+            throw ikEx;
+        } catch (Exception e) {
+            throw new InvalidKeyException(
+                    "Can not access private key components", e);
+        }
+        isValid(prKey);
+    }
+
+    /**
+     * Validate the specified RSAPublicKey
+     */
+    private void isPublicKeyValid(RSAPublicKey pKey)  throws InvalidKeyException {
+        try {
+            RSAKeyFactory.checkRSAProviderKeyLengths(
+                    pKey.getModulus().bitLength(),
+                    pKey.getPublicExponent());
+        } catch (InvalidKeyException ikEx) {
+            throw ikEx;
+        } catch (Exception e) {
+            throw new InvalidKeyException(
+                    "Can not access public key components", e);
+        }
+        isValid(pKey);
+    }
+
     /**
      * Validate the specified RSAKey and its associated parameters against
      * internal signature parameters.
      */
-    private RSAKey isValid(RSAKey rsaKey) throws InvalidKeyException {
+    private void isValid(RSAKey rsaKey) throws InvalidKeyException {
         try {
             AlgorithmParameterSpec keyParams = rsaKey.getParams();
             // validate key parameters
@@ -227,7 +279,6 @@ private RSAKey isValid(RSAKey rsaKey) throws InvalidKeyException {
                 }
                 checkKeyLength(rsaKey, hLen, this.sigParams.getSaltLength());
             }
-            return rsaKey;
         } catch (SignatureException e) {
             throw new InvalidKeyException(e);
         }

src/java.base/share/classes/sun/security/rsa/RSAPrivateCrtKeyImpl.java

@@ -80,22 +80,28 @@ public static RSAPrivateKey newKey(byte[] encoded)
         RSAPrivateCrtKeyImpl key = new RSAPrivateCrtKeyImpl(encoded);
         // check all CRT-specific components are available, if any one
         // missing, return a non-CRT key instead
-        if ((key.getPublicExponent().signum() == 0) ||
-            (key.getPrimeExponentP().signum() == 0) ||
-            (key.getPrimeExponentQ().signum() == 0) ||
-            (key.getPrimeP().signum() == 0) ||
-            (key.getPrimeQ().signum() == 0) ||
-            (key.getCrtCoefficient().signum() == 0)) {
+        if (checkComponents(key)) {
+            return key;
+        } else {
             return new RSAPrivateKeyImpl(
                 key.algid,
                 key.getModulus(),
-                key.getPrivateExponent()
-            );
-        } else {
-            return key;
+                key.getPrivateExponent());
         }
     }
 
+    /**
+     * Validate if all CRT-specific components are available.
+     */
+    static boolean checkComponents(RSAPrivateCrtKey key) {
+        return !((key.getPublicExponent().signum() == 0) ||
+            (key.getPrimeExponentP().signum() == 0) ||
+            (key.getPrimeExponentQ().signum() == 0) ||
+            (key.getPrimeP().signum() == 0) ||
+            (key.getPrimeQ().signum() == 0) ||
+            (key.getCrtCoefficient().signum() == 0));
+    }
+
     /**
      * Generate a new key from the specified type and components.
      * Returns a CRT key if possible and a non-CRT key otherwise.