8232950: SUNPKCS11 Provider incorrectly check key length for PSS Sign…

…atures.

Fixed to treat the queried key size values as bits instead of bytes

Reviewed-by: andrew
Backport-of: f14e3a60b26f0488da26abf3ae6c0521d5f616e5

jdk/src/share/classes/sun/security/pkcs11/P11PSSSignature.java

@@ -341,23 +341,23 @@ private void checkKeySize(Key key) throws InvalidKeyException {
 
         int keySize = 0; // in bytes
         if (mechInfo != null) {
-            // check against available native info
-            int minKeySize = (int) mechInfo.ulMinKeySize;
-            int maxKeySize = (int) mechInfo.ulMaxKeySize;
             if (key instanceof P11Key) {
                 keySize = (((P11Key) key).length() + 7) >> 3;
             } else if (key instanceof RSAKey) {
                 keySize = ((RSAKey) key).getModulus().bitLength() >> 3;
             } else {
                 throw new InvalidKeyException("Unrecognized key type " + key);
             }
-            if ((minKeySize != -1) && (keySize < minKeySize)) {
+            // check against available native info which are in bits
+            if ((mechInfo.iMinKeySize != 0) &&
+                    (keySize < (mechInfo.iMinKeySize >> 3))) {
                 throw new InvalidKeyException(KEY_ALGO +
-                    " key must be at least " + minKeySize + " bytes");
+                    " key must be at least " + mechInfo.iMinKeySize + " bits");
             }
-            if ((maxKeySize != -1) && (keySize > maxKeySize)) {
+            if ((mechInfo.iMaxKeySize != Integer.MAX_VALUE) &&
+                    (keySize > (mechInfo.iMaxKeySize >> 3))) {
                 throw new InvalidKeyException(KEY_ALGO +
-                    " key must be at most " + maxKeySize + " bytes");
+                    " key must be at most " + mechInfo.iMaxKeySize + " bits");
             }
         }
         if (this.sigParams != null) {