8258833: Cancel multi-part cipher operations in SunPKCS11 after failures

src/jdk.crypto.cryptoki/share/classes/sun/security/pkcs11/P11AEADCipher.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -350,6 +350,13 @@ private void cancelOperation() {
                         0, buffer, 0, bufLen);
             }
         } catch (PKCS11Exception e) {
+            if (e.getErrorCode() == CKR_OPERATION_NOT_INITIALIZED) {
+                // Cancel Operation may be invoked after an error on a PKCS#11
+                // call. If the operation inside the token was already cancelled,
+                // do not fail here. This is part of a defensive mechanism for
+                // PKCS#11 libraries that do not strictly follow the standard.
+                return;
+            }
             if (encrypt) {
                 throw new ProviderException("Cancel failed", e);
             }
@@ -616,6 +623,12 @@ private int implDoFinal(byte[] in, int inOfs, int inLen,
             }
             return k;
         } catch (PKCS11Exception e) {
+            // As per the PKCS#11 standard, C_Encrypt and C_Decrypt may only
+            // keep the operation active on CKR_BUFFER_TOO_SMALL errors or
+            // successful calls to determine the output length. However,
+            // these cases are not expected here because the output length
+            // is checked in the OpenJDK side before making the PKCS#11 call.
+            // Thus, doCancel can safely be 'false'.
             doCancel = false;
             handleException(e);
             throw new ProviderException("doFinal() failed", e);
@@ -702,6 +715,12 @@ private int implDoFinal(ByteBuffer inBuffer, ByteBuffer outBuffer)
             outBuffer.position(outBuffer.position() + k);
             return k;
         } catch (PKCS11Exception e) {
+            // As per the PKCS#11 standard, C_Encrypt and C_Decrypt may only
+            // keep the operation active on CKR_BUFFER_TOO_SMALL errors or
+            // successful calls to determine the output length. However,
+            // these cases are not expected here because the output length
+            // is checked in the OpenJDK side before making the PKCS#11 call.
+            // Thus, doCancel can safely be 'false'.
             doCancel = false;
             handleException(e);
             throw new ProviderException("doFinal() failed", e);

src/jdk.crypto.cryptoki/share/classes/sun/security/pkcs11/P11Cipher.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -439,6 +439,13 @@ private void cancelOperation() {
                 token.p11.C_DecryptFinal(session.id(), 0, buffer, 0, bufLen);
             }
         } catch (PKCS11Exception e) {
+            if (e.getErrorCode() == CKR_OPERATION_NOT_INITIALIZED) {
+                // Cancel Operation may be invoked after an error on a PKCS#11
+                // call. If the operation inside the token was already cancelled,
+                // do not fail here. This is part of a defensive mechanism for
+                // PKCS#11 libraries that do not strictly follow the standard.
+                return;
+            }
             if (encrypt) {
                 throw new ProviderException("Cancel failed", e);
             }
@@ -628,7 +635,11 @@ private int implUpdate(byte[] in, int inOfs, int inLen,
                 throw (ShortBufferException)
                         (new ShortBufferException().initCause(e));
             }
-            reset(false);
+            // Some implementations such as the NSS Software Token do not
+            // cancel the operation upon a C_EncryptUpdate/C_DecryptUpdate
+            // failure (as required by the PKCS#11 standard). See JDK-8258833
+            // for further information.
+            reset(true);
             throw new ProviderException("update() failed", e);
         }
     }
@@ -746,7 +757,11 @@ private int implUpdate(ByteBuffer inBuffer, ByteBuffer outBuffer)
                 throw (ShortBufferException)
                         (new ShortBufferException().initCause(e));
             }
-            reset(false);
+            // Some implementations such as the NSS Software Token do not
+            // cancel the operation upon a C_EncryptUpdate/C_DecryptUpdate
+            // failure (as required by the PKCS#11 standard). See JDK-8258833
+            // for further information.
+            reset(true);
             throw new ProviderException("update() failed", e);
         }
     }
@@ -770,9 +785,14 @@ private int implDoFinal(byte[] out, int outOfs, int outLen)
                             0, padBuffer, 0, actualPadLen,
                             0, out, outOfs, outLen);
                 }
+                // Some implementations such as the NSS Software Token do not
+                // cancel the operation upon a C_EncryptUpdate failure (as
+                // required by the PKCS#11 standard). Cancel is not needed
+                // only after this point. See JDK-8258833 for further
+                // information.
+                doCancel = false;
                 k += token.p11.C_EncryptFinal(session.id(),
                         0, out, (outOfs + k), (outLen - k));
-                doCancel = false;
             } else {
                 // Special handling to match SunJCE provider behavior
                 if (bytesBuffered == 0 && padBufferLen == 0) {
@@ -784,22 +804,26 @@ private int implDoFinal(byte[] out, int outOfs, int outLen)
                                 padBuffer, 0, padBufferLen, 0, padBuffer, 0,
                                 padBuffer.length);
                     }
+                    // Some implementations such as the NSS Software Token do not
+                    // cancel the operation upon a C_DecryptUpdate failure (as
+                    // required by the PKCS#11 standard). Cancel is not needed
+                    // only after this point. See JDK-8258833 for further
+                    // information.
+                    doCancel = false;
                     k += token.p11.C_DecryptFinal(session.id(), 0, padBuffer, k,
                             padBuffer.length - k);
-                    doCancel = false;
 
                     int actualPadLen = paddingObj.unpad(padBuffer, k);
                     k -= actualPadLen;
                     System.arraycopy(padBuffer, 0, out, outOfs, k);
                 } else {
+                    doCancel = false;
                     k = token.p11.C_DecryptFinal(session.id(), 0, out, outOfs,
                             outLen);
-                    doCancel = false;
                 }
             }
             return k;
         } catch (PKCS11Exception e) {
-            doCancel = false;
             handleException(e);
             throw new ProviderException("doFinal() failed", e);
         } finally {
@@ -845,9 +869,14 @@ private int implDoFinal(ByteBuffer outBuffer)
                             0, padBuffer, 0, actualPadLen,
                             outAddr, outArray, outOfs, outLen);
                 }
+                // Some implementations such as the NSS Software Token do not
+                // cancel the operation upon a C_EncryptUpdate failure (as
+                // required by the PKCS#11 standard). Cancel is not needed
+                // only after this point. See JDK-8258833 for further
+                // information.
+                doCancel = false;
                 k += token.p11.C_EncryptFinal(session.id(),
                         outAddr, outArray, (outOfs + k), (outLen - k));
-                doCancel = false;
             } else {
                 // Special handling to match SunJCE provider behavior
                 if (bytesBuffered == 0 && padBufferLen == 0) {
@@ -861,18 +890,23 @@ private int implDoFinal(ByteBuffer outBuffer)
                                 0, padBuffer, 0, padBuffer.length);
                         padBufferLen = 0;
                     }
+                    // Some implementations such as the NSS Software Token do not
+                    // cancel the operation upon a C_DecryptUpdate failure (as
+                    // required by the PKCS#11 standard). Cancel is not needed
+                    // only after this point. See JDK-8258833 for further
+                    // information.
+                    doCancel = false;
                     k += token.p11.C_DecryptFinal(session.id(),
                             0, padBuffer, k, padBuffer.length - k);
-                    doCancel = false;
 
                     int actualPadLen = paddingObj.unpad(padBuffer, k);
                     k -= actualPadLen;
                     outArray = padBuffer;
                     outOfs = 0;
                 } else {
+                    doCancel = false;
                     k = token.p11.C_DecryptFinal(session.id(),
                             outAddr, outArray, outOfs, outLen);
-                    doCancel = false;
                 }
             }
             if ((!encrypt && paddingObj != null) ||
@@ -884,7 +918,6 @@ private int implDoFinal(ByteBuffer outBuffer)
             }
             return k;
         } catch (PKCS11Exception e) {
-            doCancel = false;
             handleException(e);
             throw new ProviderException("doFinal() failed", e);
         } finally {

src/jdk.crypto.cryptoki/share/classes/sun/security/pkcs11/P11Mac.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -151,6 +151,13 @@ private void cancelOperation() {
         try {
             token.p11.C_SignFinal(session.id(), 0);
         } catch (PKCS11Exception e) {
+            if (e.getErrorCode() == CKR_OPERATION_NOT_INITIALIZED) {
+                // Cancel Operation may be invoked after an error on a PKCS#11
+                // call. If the operation inside the token was already cancelled,
+                // do not fail here. This is part of a defensive mechanism for
+                // PKCS#11 libraries that do not strictly follow the standard.
+                return;
+            }
             throw new ProviderException("Cancel failed", e);
         }
     }
@@ -213,6 +220,12 @@ protected byte[] engineDoFinal() {
             ensureInitialized();
             return token.p11.C_SignFinal(session.id(), 0);
         } catch (PKCS11Exception e) {
+            // As per the PKCS#11 standard, C_SignFinal may only
+            // keep the operation active on CKR_BUFFER_TOO_SMALL errors or
+            // successful calls to determine the output length. However,
+            // these cases are handled at OpenJDK's libj2pkcs11 native
+            // library. Thus, P11Mac::reset can be called with a 'false'
+            // doCancel argument from here.
             throw new ProviderException("doFinal() failed", e);
         } finally {
             reset(false);

src/jdk.crypto.cryptoki/share/classes/sun/security/pkcs11/P11PSSSignature.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -298,6 +298,13 @@ private void cancelOperation() {
                 }
             }
         } catch (PKCS11Exception e) {
+            if (e.getErrorCode() == CKR_OPERATION_NOT_INITIALIZED) {
+                // Cancel Operation may be invoked after an error on a PKCS#11
+                // call. If the operation inside the token was already cancelled,
+                // do not fail here. This is part of a defensive mechanism for
+                // PKCS#11 libraries that do not strictly follow the standard.
+                return;
+            }
             if (mode == M_SIGN) {
                 throw new ProviderException("cancel failed", e);
             }
@@ -662,6 +669,11 @@ protected byte[] engineSign() throws SignatureException {
             doCancel = false;
             return signature;
         } catch (PKCS11Exception pe) {
+            // As per the PKCS#11 standard, C_Sign and C_SignFinal may only
+            // keep the operation active on CKR_BUFFER_TOO_SMALL errors or
+            // successful calls to determine the output length. However,
+            // these cases are handled at OpenJDK's libj2pkcs11 native
+            // library. Thus, doCancel can safely be 'false' here.
             doCancel = false;
             throw new ProviderException(pe);
         } catch (ProviderException e) {

src/jdk.crypto.cryptoki/share/classes/sun/security/pkcs11/P11Signature.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -314,6 +314,13 @@ private void cancelOperation() {
                 }
             }
         } catch (PKCS11Exception e) {
+            if (e.getErrorCode() == CKR_OPERATION_NOT_INITIALIZED) {
+                // Cancel Operation may be invoked after an error on a PKCS#11
+                // call. If the operation inside the token was already cancelled,
+                // do not fail here. This is part of a defensive mechanism for
+                // PKCS#11 libraries that do not strictly follow the standard.
+                return;
+            }
             if (mode == M_VERIFY) {
                 long errorCode = e.getErrorCode();
                 if ((errorCode == CKR_SIGNATURE_INVALID) ||
@@ -654,6 +661,11 @@ protected byte[] engineSign() throws SignatureException {
                 }
             }
         } catch (PKCS11Exception pe) {
+            // As per the PKCS#11 standard, C_Sign and C_SignFinal may only
+            // keep the operation active on CKR_BUFFER_TOO_SMALL errors or
+            // successful calls to determine the output length. However,
+            // these cases are handled at OpenJDK's libj2pkcs11 native
+            // library. Thus, doCancel can safely be 'false' here.
             doCancel = false;
             throw new ProviderException(pe);
         } catch (SignatureException | ProviderException e) {