From 130931368f08df660a55b1c0da56a1111bd74e08 Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 24 Nov 2025 14:51:08 +0000 Subject: [PATCH 1/3] Return `ETag` header from S3 fixture AWS S3 uses the `ETag` header to identify the object contents in various API responses. `S3HttpHandler` returns this header on some paths, but not very many, and the returned header does not conform to the spec (particularly, it is not surrounded by `"` characters). This commit adds the missing response header to the `GetObject` API, fixes its format, and uses SHA256 rather than MD5 to compute the result. --- .../s3/S3BlobContainerRetriesTests.java | 20 +------ .../main/java/fixture/s3/S3HttpHandler.java | 11 +++- .../java/fixture/s3/S3HttpHandlerTests.java | 55 +++++++++++++++---- 3 files changed, 57 insertions(+), 29 deletions(-) diff --git a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java index 65b4f58f6cb55..d07fa80551a24 100644 --- a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java +++ b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java @@ -32,7 +32,6 @@ import org.elasticsearch.common.blobstore.OptionalBytesReference; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.hash.MessageDigests; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.lucene.store.ByteArrayIndexInput; import org.elasticsearch.common.lucene.store.InputStreamIndexInput; @@ -430,7 +429,7 @@ public void testWriteLargeBlob() throws Exception { assertThat(contentLength, anyOf(equalTo(lastPartSize), equalTo(bufferSize.getBytes()))); if (countDownUploads.decrementAndGet() % 2 == 0) { - exchange.getResponseHeaders().add("ETag", getBase16MD5Digest(bytes)); + exchange.getResponseHeaders().add("ETag", S3HttpHandler.getEtagFromContents(bytes)); exchange.sendResponseHeaders(HttpStatus.SC_OK, -1); exchange.close(); return; @@ -529,7 +528,7 @@ public void testWriteLargeBlobStreaming() throws Exception { if (counterUploads.incrementAndGet() % 2 == 0) { bytesReceived.addAndGet(bytes.length()); - exchange.getResponseHeaders().add("ETag", getBase16MD5Digest(bytes)); + exchange.getResponseHeaders().add("ETag", S3HttpHandler.getEtagFromContents(bytes)); exchange.sendResponseHeaders(HttpStatus.SC_OK, -1); exchange.close(); return; @@ -1382,21 +1381,6 @@ public void handle(HttpExchange exchange) throws IOException { ); } - private static String getBase16MD5Digest(BytesReference bytesReference) { - return MessageDigests.toHexString(MessageDigests.digest(bytesReference, MessageDigests.md5())); - } - - public void testGetBase16MD5Digest() { - // from Wikipedia, see also org.elasticsearch.common.hash.MessageDigestsTests.testMd5 - assertBase16MD5Digest("", "d41d8cd98f00b204e9800998ecf8427e"); - assertBase16MD5Digest("The quick brown fox jumps over the lazy dog", "9e107d9d372bb6826bd81d3542a419d6"); - assertBase16MD5Digest("The quick brown fox jumps over the lazy dog.", "e4d909c290d0fb1ca068ffaddf22cbd0"); - } - - private static void assertBase16MD5Digest(String input, String expectedDigestString) { - assertEquals(expectedDigestString, getBase16MD5Digest(new BytesArray(input))); - } - @Override protected Matcher getMaxRetriesMatcher(int maxRetries) { // some attempts make meaningful progress and do not count towards the max retry limit diff --git a/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpHandler.java b/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpHandler.java index 6ce00eef01153..aae332bb345eb 100644 --- a/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpHandler.java +++ b/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpHandler.java @@ -84,6 +84,8 @@ public S3HttpHandler(final String bucket, @Nullable final String basePath) { */ private static final Set METHODS_HAVING_NO_REQUEST_BODY = Set.of("GET", "HEAD", "DELETE"); + private static final String SHA_256_ETAG_PREFIX = "es-test-sha-256-"; + @Override public void handle(final HttpExchange exchange) throws IOException { // Remove custom query parameters before processing the request. This simulates how S3 ignores them. @@ -322,6 +324,9 @@ public void handle(final HttpExchange exchange) throws IOException { exchange.sendResponseHeaders(RestStatus.NOT_FOUND.getStatus(), -1); return; } + + exchange.getResponseHeaders().add("ETag", getEtagFromContents(blob)); + final String rangeHeader = exchange.getRequestHeaders().getFirst("Range"); if (rangeHeader == null) { exchange.getResponseHeaders().add("Content-Type", "application/octet-stream"); @@ -413,6 +418,10 @@ private boolean updateBlobContents(HttpExchange exchange, String path, BytesRefe } } + public static String getEtagFromContents(BytesReference blobContents) { + return '"' + SHA_256_ETAG_PREFIX + MessageDigests.toHexString(MessageDigests.digest(blobContents, MessageDigests.sha256())) + '"'; + } + public Map blobs() { return blobs; } @@ -490,7 +499,7 @@ private static Tuple parseRequestBody(final HttpExchange ); } } - return Tuple.tuple(MessageDigests.toHexString(MessageDigests.digest(bytesReference, MessageDigests.md5())), bytesReference); + return Tuple.tuple(getEtagFromContents(bytesReference), bytesReference); } catch (Exception e) { logger.error("exception in parseRequestBody", e); exchange.sendResponseHeaders(500, 0); diff --git a/test/fixtures/s3-fixture/src/test/java/fixture/s3/S3HttpHandlerTests.java b/test/fixtures/s3-fixture/src/test/java/fixture/s3/S3HttpHandlerTests.java index 5da274f798333..3cf2503301924 100644 --- a/test/fixtures/s3-fixture/src/test/java/fixture/s3/S3HttpHandlerTests.java +++ b/test/fixtures/s3-fixture/src/test/java/fixture/s3/S3HttpHandlerTests.java @@ -83,9 +83,12 @@ public void testSimpleObjectOperations() { false\ """); - final var body = randomAlphaOfLength(50); + final var body = new BytesArray(randomAlphaOfLength(50).getBytes(StandardCharsets.UTF_8)); assertEquals(RestStatus.OK, handleRequest(handler, "PUT", "/bucket/path/blob", body).status()); - assertEquals(new TestHttpResponse(RestStatus.OK, body), handleRequest(handler, "GET", "/bucket/path/blob")); + assertEquals( + new TestHttpResponse(RestStatus.OK, body, addETag(S3HttpHandler.getEtagFromContents(body), TestHttpExchange.EMPTY_HEADERS)), + handleRequest(handler, "GET", "/bucket/path/blob") + ); assertListObjectsResponse(handler, "", null, """ false\ @@ -135,23 +138,33 @@ public void testGetWithBytesRange() { final var blobBytes = randomBytesReference(256); assertEquals(RestStatus.OK, handleRequest(handler, "PUT", blobPath, blobBytes).status()); + final var expectedEtag = S3HttpHandler.getEtagFromContents(blobBytes); + assertEquals( "No Range", - new TestHttpResponse(RestStatus.OK, blobBytes, TestHttpExchange.EMPTY_HEADERS), + new TestHttpResponse(RestStatus.OK, blobBytes, addETag(expectedEtag, TestHttpExchange.EMPTY_HEADERS)), handleRequest(handler, "GET", blobPath) ); var end = blobBytes.length() - 1; assertEquals( "Exact Range: bytes=0-" + end, - new TestHttpResponse(RestStatus.PARTIAL_CONTENT, blobBytes, contentRangeHeader(0, end, blobBytes.length())), + new TestHttpResponse( + RestStatus.PARTIAL_CONTENT, + blobBytes, + addETag(expectedEtag, contentRangeHeader(0, end, blobBytes.length())) + ), handleRequest(handler, "GET", blobPath, BytesArray.EMPTY, bytesRangeHeader(0, end)) ); end = randomIntBetween(blobBytes.length() - 1, Integer.MAX_VALUE); assertEquals( "Larger Range: bytes=0-" + end, - new TestHttpResponse(RestStatus.PARTIAL_CONTENT, blobBytes, contentRangeHeader(0, blobBytes.length() - 1, blobBytes.length())), + new TestHttpResponse( + RestStatus.PARTIAL_CONTENT, + blobBytes, + addETag(expectedEtag, contentRangeHeader(0, blobBytes.length() - 1, blobBytes.length())) + ), handleRequest(handler, "GET", blobPath, BytesArray.EMPTY, bytesRangeHeader(0, end)) ); @@ -159,7 +172,11 @@ public void testGetWithBytesRange() { end = randomIntBetween(start, Integer.MAX_VALUE); assertEquals( "Invalid Range: bytes=" + start + '-' + end, - new TestHttpResponse(RestStatus.REQUESTED_RANGE_NOT_SATISFIED, BytesArray.EMPTY, TestHttpExchange.EMPTY_HEADERS), + new TestHttpResponse( + RestStatus.REQUESTED_RANGE_NOT_SATISFIED, + BytesArray.EMPTY, + addETag(expectedEtag, TestHttpExchange.EMPTY_HEADERS) + ), handleRequest(handler, "GET", blobPath, BytesArray.EMPTY, bytesRangeHeader(start, end)) ); @@ -167,7 +184,7 @@ public void testGetWithBytesRange() { end = randomIntBetween(0, start - 1); assertEquals( "Weird Valid Range: bytes=" + start + '-' + end, - new TestHttpResponse(RestStatus.OK, blobBytes, TestHttpExchange.EMPTY_HEADERS), + new TestHttpResponse(RestStatus.OK, blobBytes, addETag(expectedEtag, TestHttpExchange.EMPTY_HEADERS)), handleRequest(handler, "GET", blobPath, BytesArray.EMPTY, bytesRangeHeader(start, end)) ); @@ -179,7 +196,7 @@ public void testGetWithBytesRange() { new TestHttpResponse( RestStatus.PARTIAL_CONTENT, blobBytes.slice(start, length), - contentRangeHeader(start, end, blobBytes.length()) + addETag(expectedEtag, contentRangeHeader(start, end, blobBytes.length())) ), handleRequest(handler, "GET", blobPath, BytesArray.EMPTY, bytesRangeHeader(start, end)) ); @@ -245,7 +262,15 @@ public void testSingleMultipartUpload() { path/blob100\ """); - assertEquals(new TestHttpResponse(RestStatus.OK, part1 + part2), handleRequest(handler, "GET", "/bucket/path/blob")); + final var expectedContents = new BytesArray((part1 + part2).getBytes(StandardCharsets.UTF_8)); + assertEquals( + new TestHttpResponse( + RestStatus.OK, + expectedContents, + addETag(S3HttpHandler.getEtagFromContents(expectedContents), TestHttpExchange.EMPTY_HEADERS) + ), + handleRequest(handler, "GET", "/bucket/path/blob") + ); assertEquals(new TestHttpResponse(RestStatus.OK, """ \ @@ -416,7 +441,11 @@ public void testPreventObjectOverwrite() throws InterruptedException { }); assertEquals( - new TestHttpResponse(RestStatus.OK, successfulTasks.getFirst().body, TestHttpExchange.EMPTY_HEADERS), + new TestHttpResponse( + RestStatus.OK, + successfulTasks.getFirst().body, + addETag(S3HttpHandler.getEtagFromContents(successfulTasks.getFirst().body), TestHttpExchange.EMPTY_HEADERS) + ), handleRequest(handler, "GET", "/bucket/path/blob") ); } @@ -562,6 +591,12 @@ private static Headers ifNoneMatchHeader() { return headers; } + private static Headers addETag(String eTag, Headers headers) { + final var newHeaders = new Headers(headers); + newHeaders.add("ETag", eTag); + return newHeaders; + } + private static class TestHttpExchange extends HttpExchange { private static final Headers EMPTY_HEADERS = new Headers(); From 6e98292a276e4c7ce31a69ea175c5be7405c99d6 Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 24 Nov 2025 15:56:50 +0000 Subject: [PATCH 2/3] Reinstate canned-value hash tests --- .../test/java/fixture/s3/S3HttpHandlerTests.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/fixtures/s3-fixture/src/test/java/fixture/s3/S3HttpHandlerTests.java b/test/fixtures/s3-fixture/src/test/java/fixture/s3/S3HttpHandlerTests.java index 3cf2503301924..ec7a3e15da707 100644 --- a/test/fixtures/s3-fixture/src/test/java/fixture/s3/S3HttpHandlerTests.java +++ b/test/fixtures/s3-fixture/src/test/java/fixture/s3/S3HttpHandlerTests.java @@ -488,6 +488,20 @@ private static TestWriteTask createMultipartUploadTask(S3HttpHandler handler) { return multipartUploadTask; } + public void testGetETagFromContents() { + // empty-string value from Wikipedia, see also org.elasticsearch.common.hash.MessageDigestsTests.testSha256 + assertETag("", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); + assertETag("The quick brown fox jumps over the lazy dog", "d7a8fbb307d7809469ca9abcb0082e4f8d5651e46d3cdb762d02d0bf37c9e592"); + assertETag("The quick brown fox jumps over the lazy cog", "e4c4d8f3bf76b692de791a173e05321150f7a345b46484fe427f6acc7ecc81be"); + } + + private static void assertETag(String input, String expectedHash) { + assertEquals( + "\"es-test-sha-256-" + expectedHash + '"', + S3HttpHandler.getEtagFromContents(new BytesArray(input.getBytes(StandardCharsets.UTF_8))) + ); + } + private static class TestWriteTask { final BytesReference body; final Runnable consumer; From da4e70e364fd715e0a1d27b24071c9046f4d926c Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 24 Nov 2025 16:07:14 +0000 Subject: [PATCH 3/3] Add comment --- .../s3-fixture/src/main/java/fixture/s3/S3HttpHandler.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpHandler.java b/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpHandler.java index aae332bb345eb..619582ba7ab9d 100644 --- a/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpHandler.java +++ b/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpHandler.java @@ -418,6 +418,11 @@ private boolean updateBlobContents(HttpExchange exchange, String path, BytesRefe } } + /** + * Etags are opaque identifiers for the contents of an object. + * + * @see HTTP ETag on Wikipedia. + */ public static String getEtagFromContents(BytesReference blobContents) { return '"' + SHA_256_ETAG_PREFIX + MessageDigests.toHexString(MessageDigests.digest(blobContents, MessageDigests.sha256())) + '"'; }