8271199: Mutual TLS handshake fails signing client certificate with c…

…ustom sensitive PKCS11 key

Reviewed-by: xuelei, valeriep

src/java.base/share/classes/sun/security/rsa/RSAPSSSignature.java

@@ -123,12 +123,14 @@ public RSAPSSSignature() {
     @Override
     protected void engineInitVerify(PublicKey publicKey)
             throws InvalidKeyException {
-        if (!(publicKey instanceof RSAPublicKey)) {
+        if (publicKey instanceof RSAPublicKey rsaPubKey) {
+            isPublicKeyValid(rsaPubKey);
+            this.pubKey = rsaPubKey;
+            this.privKey = null;
+            resetDigest();
+        } else {
             throw new InvalidKeyException("key must be RSAPublicKey");
         }
-        this.pubKey = (RSAPublicKey) isValid((RSAKey)publicKey);
-        this.privKey = null;
-        resetDigest();
     }
 
     // initialize for signing. See JCA doc
@@ -142,14 +144,16 @@ protected void engineInitSign(PrivateKey privateKey)
     @Override
     protected void engineInitSign(PrivateKey privateKey, SecureRandom random)
             throws InvalidKeyException {
-        if (!(privateKey instanceof RSAPrivateKey)) {
+        if (privateKey instanceof RSAPrivateKey rsaPrivateKey) {
+            isPrivateKeyValid(rsaPrivateKey);
+            this.privKey = rsaPrivateKey;
+            this.pubKey = null;
+            this.random =
+                    (random == null ? JCAUtil.getSecureRandom() : random);
+            resetDigest();
+        } else {
             throw new InvalidKeyException("key must be RSAPrivateKey");
         }
-        this.privKey = (RSAPrivateKey) isValid((RSAKey)privateKey);
-        this.pubKey = null;
-        this.random =
-            (random == null? JCAUtil.getSecureRandom() : random);
-        resetDigest();
     }
 
     /**
@@ -201,11 +205,56 @@ private static boolean isCompatible(AlgorithmParameterSpec keyParams,
         }
     }
 
+    /**
+     * Validate the specified RSAPrivateKey
+     */
+    private void isPrivateKeyValid(RSAPrivateKey prKey)  throws InvalidKeyException {
+        try {
+            if (prKey instanceof RSAPrivateCrtKey crtKey) {
+                if (RSAPrivateCrtKeyImpl.checkComponents(crtKey)) {
+                    RSAKeyFactory.checkRSAProviderKeyLengths(
+                            crtKey.getModulus().bitLength(),
+                            crtKey.getPublicExponent());
+                } else {
+                    throw new InvalidKeyException(
+                            "Some of the CRT-specific components are not available");
+                }
+            } else {
+                RSAKeyFactory.checkRSAProviderKeyLengths(
+                        prKey.getModulus().bitLength(),
+                        null);
+            }
+        } catch (InvalidKeyException ikEx) {
+            throw ikEx;
+        } catch (Exception e) {
+            throw new InvalidKeyException(
+                    "Can not access private key components", e);
+        }
+        isValid(prKey);
+    }
+
+    /**
+     * Validate the specified RSAPublicKey
+     */
+    private void isPublicKeyValid(RSAPublicKey pKey)  throws InvalidKeyException {
+        try {
+            RSAKeyFactory.checkRSAProviderKeyLengths(
+                    pKey.getModulus().bitLength(),
+                    pKey.getPublicExponent());
+        } catch (InvalidKeyException ikEx) {
+            throw ikEx;
+        } catch (Exception e) {
+            throw new InvalidKeyException(
+                    "Can not access public key components", e);
+        }
+        isValid(pKey);
+    }
+
     /**
      * Validate the specified RSAKey and its associated parameters against
      * internal signature parameters.
      */
-    private RSAKey isValid(RSAKey rsaKey) throws InvalidKeyException {
+    private void isValid(RSAKey rsaKey) throws InvalidKeyException {
         AlgorithmParameterSpec keyParams = rsaKey.getParams();
         // validate key parameters
         if (!isCompatible(rsaKey.getParams(), this.sigParams)) {
@@ -232,7 +281,6 @@ private RSAKey isValid(RSAKey rsaKey) throws InvalidKeyException {
                         ("Unrecognized digest algo: " + digestAlgo);
             }
         }
-        return rsaKey;
     }
 
     /**

src/java.base/share/classes/sun/security/rsa/RSAPrivateCrtKeyImpl.java

@@ -91,16 +91,11 @@ public static RSAPrivateKey newKey(KeyType type, String format,
             RSAKeyFactory.checkKeyAlgo(key, type.keyAlgo);
             // check all CRT-specific components are available, if any one
             // missing, return a non-CRT key instead
-            if ((key.getPublicExponent().signum() == 0) ||
-                (key.getPrimeExponentP().signum() == 0) ||
-                (key.getPrimeExponentQ().signum() == 0) ||
-                (key.getPrimeP().signum() == 0) ||
-                (key.getPrimeQ().signum() == 0) ||
-                (key.getCrtCoefficient().signum() == 0)) {
+            if (checkComponents(key)) {
+                return key;
+            } else {
                 return new RSAPrivateKeyImpl(key.type, key.keyParams,
                     key.getModulus(), key.getPrivateExponent());
-            } else {
-                return key;
             }
         case "PKCS#1":
             try {
@@ -124,6 +119,18 @@ public static RSAPrivateKey newKey(KeyType type, String format,
         }
     }
 
+    /**
+     * Validate if all CRT-specific components are available.
+     */
+    static boolean checkComponents(RSAPrivateCrtKey key) {
+        return !((key.getPublicExponent().signum() == 0) ||
+            (key.getPrimeExponentP().signum() == 0) ||
+            (key.getPrimeExponentQ().signum() == 0) ||
+            (key.getPrimeP().signum() == 0) ||
+            (key.getPrimeQ().signum() == 0) ||
+            (key.getCrtCoefficient().signum() == 0));
+    }
+
     /**
      * Generate a new key from the specified type and components.
      * Returns a CRT key if possible and a non-CRT key otherwise.