From bd46eb5688a05e2b7bed56d18b18987075984f88 Mon Sep 17 00:00:00 2001 From: David Turner Date: Wed, 2 Apr 2025 08:41:08 +0100 Subject: [PATCH] Migrate tests away from S3 SDK `MD5DigestCalculatingInputStream` `S3BlobContainerRetriesTests` uses `MD5DigestCalculatingInputStream` from the AWS v1 SDK to compute a MD5 checksum, but this feature is not available in the v2 SDK. With this commit we remove this dependency and compute the MD5 checksums directly instead. --- .../s3/S3BlobContainerRetriesTests.java | 29 ++++++++++++++----- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java index a3f2ca10bfd06..fe8c0ec2c341d 100644 --- a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java +++ b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java @@ -14,9 +14,7 @@ import com.amazonaws.DnsResolver; import com.amazonaws.SdkClientException; import com.amazonaws.services.s3.AmazonS3ClientBuilder; -import com.amazonaws.services.s3.internal.MD5DigestCalculatingInputStream; import com.amazonaws.services.s3.model.AmazonS3Exception; -import com.amazonaws.util.Base16; import com.sun.net.httpserver.HttpExchange; import com.sun.net.httpserver.HttpHandler; @@ -32,7 +30,9 @@ import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.blobstore.OperationPurpose; import org.elasticsearch.common.blobstore.OptionalBytesReference; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.hash.MessageDigests; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.lucene.store.ByteArrayIndexInput; import org.elasticsearch.common.lucene.store.InputStreamIndexInput; @@ -431,13 +431,12 @@ public void testWriteLargeBlob() throws Exception { } } else if (s3Request.isUploadPartRequest()) { // upload part request - MD5DigestCalculatingInputStream md5 = new MD5DigestCalculatingInputStream(exchange.getRequestBody()); - BytesReference bytes = Streams.readFully(md5); + BytesReference bytes = Streams.readFully(exchange.getRequestBody()); assertThat((long) bytes.length(), anyOf(equalTo(lastPartSize), equalTo(bufferSize.getBytes()))); assertThat(contentLength, anyOf(equalTo(lastPartSize), equalTo(bufferSize.getBytes()))); if (countDownUploads.decrementAndGet() % 2 == 0) { - exchange.getResponseHeaders().add("ETag", Base16.encodeAsString(md5.getMd5Digest())); + exchange.getResponseHeaders().add("ETag", getBase16MD5Digest(bytes)); exchange.sendResponseHeaders(HttpStatus.SC_OK, -1); exchange.close(); return; @@ -529,12 +528,11 @@ public void testWriteLargeBlobStreaming() throws Exception { } } else if (s3Request.isUploadPartRequest()) { // upload part request - MD5DigestCalculatingInputStream md5 = new MD5DigestCalculatingInputStream(exchange.getRequestBody()); - BytesReference bytes = Streams.readFully(md5); + BytesReference bytes = Streams.readFully(exchange.getRequestBody()); if (counterUploads.incrementAndGet() % 2 == 0) { bytesReceived.addAndGet(bytes.length()); - exchange.getResponseHeaders().add("ETag", Base16.encodeAsString(md5.getMd5Digest())); + exchange.getResponseHeaders().add("ETag", getBase16MD5Digest(bytes)); exchange.sendResponseHeaders(HttpStatus.SC_OK, -1); exchange.close(); return; @@ -1159,6 +1157,21 @@ public void testTrimmedLogAndCappedSuppressedErrorOnMultiObjectDeletionException } } + private static String getBase16MD5Digest(BytesReference bytesReference) { + return MessageDigests.toHexString(MessageDigests.digest(bytesReference, MessageDigests.md5())); + } + + public void testGetBase16MD5Digest() { + // from Wikipedia, see also org.elasticsearch.common.hash.MessageDigestsTests.testMd5 + assertBase16MD5Digest("", "d41d8cd98f00b204e9800998ecf8427e"); + assertBase16MD5Digest("The quick brown fox jumps over the lazy dog", "9e107d9d372bb6826bd81d3542a419d6"); + assertBase16MD5Digest("The quick brown fox jumps over the lazy dog.", "e4d909c290d0fb1ca068ffaddf22cbd0"); + } + + private static void assertBase16MD5Digest(String input, String expectedDigestString) { + assertEquals(expectedDigestString, getBase16MD5Digest(new BytesArray(input))); + } + @Override protected Matcher getMaxRetriesMatcher(int maxRetries) { // some attempts make meaningful progress and do not count towards the max retry limit