Skip to content

Commit 479ddb6

Browse files
Scott GibbonsRealCLanger
Scott Gibbons
authored andcommitted
8280703: CipherCore.doFinal(...) causes potentially massive byte[] allocations during decryption
Reviewed-by: clanger Backport-of: 409382ba4b43bf48ed0086020dd20641effd35b6
1 parent 7d89919 commit 479ddb6

File tree

1 file changed

+16
-10
lines changed

1 file changed

+16
-10
lines changed

src/java.base/share/classes/com/sun/crypto/provider/CipherCore.java

+16-10
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2002, 2018, Oracle and/or its affiliates. All rights reserved.
2+
* Copyright (c) 2002, 2022, Oracle and/or its affiliates. All rights reserved.
33
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
44
*
55
* This code is free software; you can redistribute it and/or modify it
@@ -915,10 +915,10 @@ int doFinal(byte[] input, int inputOffset, int inputLen, byte[] output,
915915
int estOutSize = getOutputSizeByOperation(inputLen, true);
916916
int outputCapacity = checkOutputCapacity(output, outputOffset,
917917
estOutSize);
918-
int offset = decrypting ? 0 : outputOffset; // 0 for decrypting
918+
int offset = outputOffset; // 0 for decrypting
919919
byte[] finalBuf = prepareInputBuffer(input, inputOffset,
920920
inputLen, output, outputOffset);
921-
byte[] outWithPadding = null; // for decrypting only
921+
byte[] internalOutput = null; // for decrypting only
922922

923923
int finalOffset = (finalBuf == input) ? inputOffset : 0;
924924
int finalBufLen = (finalBuf == input) ? inputLen : finalBuf.length;
@@ -932,11 +932,15 @@ int doFinal(byte[] input, int inputOffset, int inputLen, byte[] output,
932932
if (outputCapacity < estOutSize) {
933933
cipher.save();
934934
}
935-
// create temporary output buffer so that only "real"
936-
// data bytes are passed to user's output buffer.
937-
outWithPadding = new byte[estOutSize];
935+
if (outputCapacity < estOutSize || padding != null) {
936+
// create temporary output buffer if the estimated size is larger
937+
// than the user-provided buffer or a padding needs to be removed
938+
// before copying the unpadded result to the output buffer
939+
internalOutput = new byte[estOutSize];
940+
offset = 0;
941+
}
938942
}
939-
byte[] outBuffer = decrypting ? outWithPadding : output;
943+
byte[] outBuffer = (internalOutput != null) ? internalOutput : output;
940944

941945
int outLen = fillOutputBuffer(finalBuf, finalOffset, outBuffer,
942946
offset, finalBufLen, input);
@@ -952,9 +956,11 @@ int doFinal(byte[] input, int inputOffset, int inputLen, byte[] output,
952956
+ " bytes needed");
953957
}
954958
// copy the result into user-supplied output buffer
955-
System.arraycopy(outWithPadding, 0, output, outputOffset, outLen);
956-
// decrypt mode. Zero out output data that's not required
957-
Arrays.fill(outWithPadding, (byte) 0x00);
959+
if (internalOutput != null) {
960+
System.arraycopy(internalOutput, 0, output, outputOffset, outLen);
961+
// decrypt mode. Zero out output data that's not required
962+
Arrays.fill(internalOutput, (byte) 0x00);
963+
}
958964
}
959965
endDoFinal();
960966
return outLen;

0 commit comments

Comments
 (0)