8229243: SunPKCS11-Solaris provider tests failing on Solaris 11.4

For CK_GCM_PARAMS, try the spec definition first before falling back to the header file definition

Backport-of: 381e90e

src/jdk.crypto.cryptoki/share/classes/sun/security/pkcs11/P11AEADCipher.java

@@ -378,9 +378,6 @@ private void initialize() throws PKCS11Exception {
 
         long p11KeyID = p11Key.getKeyID();
         try {
-            if (session == null) {
-                session = token.getOpSession();
-            }
             CK_MECHANISM mechWithParams;
             switch (blockMode) {
                 case MODE_GCM:
@@ -390,6 +387,9 @@ private void initialize() throws PKCS11Exception {
                 default:
                     throw new ProviderException("Unsupported mode: " + blockMode);
             }
+            if (session == null) {
+                session = token.getOpSession();
+            }
             if (encrypt) {
                 token.p11.C_EncryptInit(session.id(), mechWithParams,
                     p11KeyID);
@@ -398,7 +398,6 @@ private void initialize() throws PKCS11Exception {
                     p11KeyID);
             }
         } catch (PKCS11Exception e) {
-            //e.printStackTrace();
             p11Key.releaseKeyID();
             session = token.releaseSession(session);
             throw e;
@@ -718,7 +717,9 @@ private void handleException(PKCS11Exception e)
                    errorCode == CKR_ENCRYPTED_DATA_LEN_RANGE) {
             throw (IllegalBlockSizeException)
                     (new IllegalBlockSizeException(e.toString()).initCause(e));
-        } else if (errorCode == CKR_ENCRYPTED_DATA_INVALID) {
+        } else if (errorCode == CKR_ENCRYPTED_DATA_INVALID ||
+                // Solaris-specific
+                errorCode == CKR_GENERAL_ERROR) {
             throw (BadPaddingException)
                     (new BadPaddingException(e.toString()).initCause(e));
         }

src/jdk.crypto.cryptoki/share/classes/sun/security/pkcs11/P11Digest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -103,9 +103,11 @@ final class P11Digest extends MessageDigestSpi implements Cloneable,
             digestLength = 20;
             break;
         case (int)CKM_SHA224:
+        case (int)CKM_SHA512_224:
             digestLength = 28;
             break;
         case (int)CKM_SHA256:
+        case (int)CKM_SHA512_256:
             digestLength = 32;
             break;
         case (int)CKM_SHA384:

src/jdk.crypto.cryptoki/share/classes/sun/security/pkcs11/P11Mac.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -91,9 +91,11 @@ final class P11Mac extends MacSpi {
             macLength = 20;
             break;
         case (int)CKM_SHA224_HMAC:
+        case (int)CKM_SHA512_224_HMAC:
             macLength = 28;
             break;
         case (int)CKM_SHA256_HMAC:
+        case (int)CKM_SHA512_256_HMAC:
             macLength = 32;
             break;
         case (int)CKM_SHA384_HMAC:

src/jdk.crypto.cryptoki/share/native/libj2pkcs11/p11_convert.c

@@ -721,7 +721,7 @@ jTlsMacParamsToCKTlsMacParamPtr(JNIEnv *env, jobject jParam, CK_ULONG *pLength)
     }
 
     // populate using java values
-    ckParamPtr->prfMechanism = jLongToCKULong(jPrfMechanism);
+    ckParamPtr->prfHashMechanism = jLongToCKULong(jPrfMechanism);
     ckParamPtr->ulMacLength = jLongToCKULong(jUlMacLength);
     ckParamPtr->ulServerOrClient = jLongToCKULong(jUlServerOrClient);
 
@@ -1014,17 +1014,18 @@ jAesCtrParamsToCKAesCtrParamPtr(JNIEnv *env, jobject jParam, CK_ULONG *pLength)
 }
 
 /*
- * converts the Java CK_GCM_PARAMS object to a CK_GCM_PARAMS pointer
+ * converts the Java CK_GCM_PARAMS object to a CK_GCM_PARAMS_NO_IVBITS pointer
+ * Note: Need to try NSS definition first to avoid SIGSEGV.
  *
  * @param env - used to call JNI funktions to get the Java classes and objects
  * @param jParam - the Java CK_GCM_PARAMS object to convert
  * @param pLength - length of the allocated memory of the returned pointer
- * @return pointer to the new CK_GCM_PARAMS structure
+ * @return pointer to the new CK_GCM_PARAMS_NO_IVBITS structure
  */
-CK_GCM_PARAMS_PTR
+CK_GCM_PARAMS_NO_IVBITS_PTR
 jGCMParamsToCKGCMParamPtr(JNIEnv *env, jobject jParam, CK_ULONG *pLength)
 {
-    CK_GCM_PARAMS_PTR ckParamPtr;
+    CK_GCM_PARAMS_NO_IVBITS_PTR ckParamPtr;
     jclass jGcmParamsClass;
     jfieldID fieldID;
     jobject jIv, jAad;
@@ -1052,8 +1053,8 @@ jGCMParamsToCKGCMParamPtr(JNIEnv *env, jobject jParam, CK_ULONG *pLength)
     if (fieldID == NULL) { return NULL; }
     jTagLen = (*env)->GetLongField(env, jParam, fieldID);
 
-    // allocate memory for CK_GCM_PARAMS pointer
-    ckParamPtr = calloc(1, sizeof(CK_GCM_PARAMS));
+    // allocate memory for CK_GCM_PARAMS_NO_IVBITS pointer
+    ckParamPtr = calloc(1, sizeof(CK_GCM_PARAMS_NO_IVBITS));
     if (ckParamPtr == NULL) {
         throwOutOfMemoryError(env, 0);
         return NULL;
@@ -1073,16 +1074,15 @@ jGCMParamsToCKGCMParamPtr(JNIEnv *env, jobject jParam, CK_ULONG *pLength)
     ckParamPtr->ulTagBits = jLongToCKULong(jTagLen);
 
     if (pLength != NULL) {
-        *pLength = sizeof(CK_GCM_PARAMS);
+        *pLength = sizeof(CK_GCM_PARAMS_NO_IVBITS);
     }
-    TRACE1("Created inner GCM_PARAMS PTR %lX\n", ptr_to_jlong(ckParamPtr));
+    TRACE1("Created inner GCM_PARAMS PTR w/o ulIvBits %p\n", ckParamPtr);
     return ckParamPtr;
 cleanup:
     free(ckParamPtr->pIv);
     free(ckParamPtr->pAAD);
     free(ckParamPtr);
     return NULL;
-
 }
 
 /*
@@ -1179,7 +1179,7 @@ CK_MECHANISM_PTR jMechanismToCKMechanismPtr(JNIEnv *env, jobject jMech)
         throwOutOfMemoryError(env, 0);
         return NULL;
     }
-    TRACE1("DEBUG jMechanismToCKMechanismPtr: allocated mech %p \n", ckpMech);
+    TRACE1("DEBUG jMechanismToCKMechanismPtr: allocated mech %p\n", ckpMech);
 
     ckpMech->mechanism = jLongToCKULong(jMechType);
 

src/jdk.crypto.cryptoki/share/native/libj2pkcs11/p11_crypt.c

@@ -72,6 +72,7 @@ Java_sun_security_pkcs11_wrapper_PKCS11_C_1EncryptInit
 {
     CK_SESSION_HANDLE ckSessionHandle;
     CK_MECHANISM_PTR ckpMechanism = NULL;
+    CK_MECHANISM_PTR ckpTemp;
     CK_OBJECT_HANDLE ckKeyHandle;
     CK_RV rv;
 
@@ -81,15 +82,32 @@ Java_sun_security_pkcs11_wrapper_PKCS11_C_1EncryptInit
     ckSessionHandle = jLongToCKULong(jSessionHandle);
     ckKeyHandle = jLongToCKULong(jKeyHandle);
     ckpMechanism = jMechanismToCKMechanismPtr(env, jMechanism);
+    TRACE1("DEBUG C_EncryptInit: created pMech = %p\n",
+            ckpMechanism);
+
     if ((*env)->ExceptionCheck(env)) { return; }
 
     rv = (*ckpFunctions->C_EncryptInit)(ckSessionHandle, ckpMechanism,
                                         ckKeyHandle);
 
-    // if OAEP, then cannot free here
-    freeCKMechanismPtr(ckpMechanism);
+    if (ckpMechanism->mechanism == CKM_AES_GCM) {
+        if (rv == CKR_ARGUMENTS_BAD || rv == CKR_MECHANISM_PARAM_INVALID) {
+            // retry with CKM_GCM_PARAMS structure in pkcs11t.h
+            TRACE0("DEBUG C_EncryptInit: retry with CK_GCM_PARAMS\n");
+            ckpTemp = updateGCMParams(env, ckpMechanism);
+            if (ckpTemp != NULL) { // only re-call if conversion succeeds
+                ckpMechanism = ckpTemp;
+                rv = (*ckpFunctions->C_EncryptInit)(ckSessionHandle, ckpMechanism,
+                        ckKeyHandle);
+            }
+        }
+    }
 
+    TRACE1("DEBUG C_EncryptInit: freed pMech = %p\n", ckpMechanism);
+    freeCKMechanismPtr(ckpMechanism);
     if (ckAssertReturnValueOK(env, rv) != CK_ASSERT_OK) { return; }
+
+    TRACE0("FINISHED\n");
 }
 #endif
 
@@ -292,6 +310,7 @@ Java_sun_security_pkcs11_wrapper_PKCS11_C_1DecryptInit
 {
     CK_SESSION_HANDLE ckSessionHandle;
     CK_MECHANISM_PTR ckpMechanism = NULL;
+    CK_MECHANISM_PTR ckpTemp;
     CK_OBJECT_HANDLE ckKeyHandle;
     CK_RV rv;
 
@@ -301,15 +320,32 @@ Java_sun_security_pkcs11_wrapper_PKCS11_C_1DecryptInit
     ckSessionHandle = jLongToCKULong(jSessionHandle);
     ckKeyHandle = jLongToCKULong(jKeyHandle);
     ckpMechanism = jMechanismToCKMechanismPtr(env, jMechanism);
+    TRACE1("DEBUG C_DecryptInit: created pMech = %p\n",
+            ckpMechanism);
+
     if ((*env)->ExceptionCheck(env)) { return; }
 
     rv = (*ckpFunctions->C_DecryptInit)(ckSessionHandle, ckpMechanism,
                                         ckKeyHandle);
 
-    // if OAEP, then cannot free here
-    freeCKMechanismPtr(ckpMechanism);
+    if (ckpMechanism->mechanism == CKM_AES_GCM) {
+        if (rv == CKR_ARGUMENTS_BAD || rv == CKR_MECHANISM_PARAM_INVALID) {
+            // retry with CKM_GCM_PARAMS structure in pkcs11t.h
+            TRACE0("DEBUG C_DecryptInit: retry with CK_GCM_PARAMS\n");
+            ckpTemp = updateGCMParams(env, ckpMechanism);
+            if (ckpTemp != NULL) { // only re-call if conversion succeeds
+                ckpMechanism = ckpTemp;
+                rv = (*ckpFunctions->C_DecryptInit)(ckSessionHandle, ckpMechanism,
+                        ckKeyHandle);
+            }
+        }
+    }
 
+    TRACE1("DEBUG C_DecryptInit: freed pMech = %p\n", ckpMechanism);
+    freeCKMechanismPtr(ckpMechanism);
     if (ckAssertReturnValueOK(env, rv) != CK_ASSERT_OK) { return; }
+
+    TRACE0("FINISHED\n");
 }
 #endif