diff --git a/README.md b/README.md index af8750b..b93622d 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ -## FusionAuth HTTP client and server ![semver 2.0.0 compliant](http://img.shields.io/badge/semver-2.0.0-brightgreen.svg?style=flat-square) [![test](https://github.com/FusionAuth/java-http/actions/workflows/test.yml/badge.svg)](https://github.com/FusionAuth/java-http/actions/workflows/test.yml) +## Java HTTP client and server ![semver 2.0.0 compliant](http://img.shields.io/badge/semver-2.0.0-brightgreen.svg?style=flat-square) [![test](https://github.com/FusionAuth/java-http/actions/workflows/test.yml/badge.svg)](https://github.com/FusionAuth/java-http/actions/workflows/test.yml) ### Latest versions -* Latest stable version: `1.3.1` +* Latest stable version: `1.4.0` * Now with 100% more virtual threads! * Prior stable version `0.3.7` @@ -27,20 +27,20 @@ To add this library to your project, you can include this dependency in your Mav io.fusionauth java-http - 1.3.1 + 1.4.0 ``` If you are using Gradle, you can add this to your build file: ```groovy -implementation 'io.fusionauth:java-http:1.3.1' +implementation 'io.fusionauth:java-http:1.4.0' ``` If you are using Savant, you can add this to your build file: ```groovy -dependency(id: "io.fusionauth:java-http:1.3.1") +dependency(id: "io.fusionauth:java-http:1.4.0") ``` ## Examples Usages: @@ -231,14 +231,13 @@ The general requirements and roadmap are as follows: ### Server tasks * [x] Basic HTTP 1.1 +* [x] Support Accept-Encoding (gzip, deflate), by default and per response options. +* [x] Support Content-Encoding (gzip, deflate) * [x] Support Keep-Alive * [x] Support Expect-Continue 100 -* [x] Support chunked request -* [x] Support chunked response -* [x] Support streaming entity bodies (via chunking likely) -* [x] Support compression (default and per response options) +* [x] Support Transfer-Encoding: chunked on request for streaming. +* [x] Support Transfer-Encoding: chunked on response * [x] Support cookies in request and response -* [x] Clean up HTTPRequest * [x] Support form data * [x] Support multipart form data * [x] Support TLS diff --git a/build.savant b/build.savant index 7f9cbdf..5ee435d 100644 --- a/build.savant +++ b/build.savant @@ -18,7 +18,7 @@ restifyVersion = "4.2.1" slf4jVersion = "2.0.17" testngVersion = "7.11.0" -project(group: "io.fusionauth", name: "java-http", version: "1.3.1", licenses: ["ApacheV2_0"]) { +project(group: "io.fusionauth", name: "java-http", version: "1.4.0", licenses: ["ApacheV2_0"]) { workflow { fetch { // Dependency resolution order: diff --git a/pom.xml b/pom.xml index 3b20e07..cb98b92 100644 --- a/pom.xml +++ b/pom.xml @@ -2,7 +2,7 @@ 4.0.0 io.fusionauth java-http - 1.3.1 + 1.4.0 jar Java HTTP library (client and server) @@ -200,4 +200,4 @@ - \ No newline at end of file + diff --git a/src/main/java/io/fusionauth/http/HTTPValues.java b/src/main/java/io/fusionauth/http/HTTPValues.java index 42697ab..7f21312 100644 --- a/src/main/java/io/fusionauth/http/HTTPValues.java +++ b/src/main/java/io/fusionauth/http/HTTPValues.java @@ -51,6 +51,8 @@ public static final class ContentEncodings { public static final String Gzip = "gzip"; + public static final String XGzip = "x-gzip"; + private ContentEncodings() { } } @@ -216,6 +218,8 @@ public static final class Headers { public static final String ContentEncoding = "Content-Encoding"; + public static final String ContentEncodingLower = "content-encoding"; + public static final String ContentLength = "Content-Length"; public static final String ContentLengthLower = "content-length"; diff --git a/src/main/java/io/fusionauth/http/io/FixedLengthInputStream.java b/src/main/java/io/fusionauth/http/io/FixedLengthInputStream.java new file mode 100644 index 0000000..0b411a3 --- /dev/null +++ b/src/main/java/io/fusionauth/http/io/FixedLengthInputStream.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2025, FusionAuth, All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the License. + */ +package io.fusionauth.http.io; + +import java.io.IOException; +import java.io.InputStream; + +/** + * A filter InputStream that reads a fixed length body. + * + * @author Daniel DeGroff + */ +public class FixedLengthInputStream extends InputStream { + private final byte[] b1 = new byte[1]; + + private final PushbackInputStream delegate; + + private long bytesRemaining; + + public FixedLengthInputStream(PushbackInputStream delegate, long contentLength) { + this.delegate = delegate; + this.bytesRemaining = contentLength; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + if (bytesRemaining <= 0) { + return -1; + } + + int read = delegate.read(b, off, len); + int reportBytesRead = read; + if (read > 0) { + int extraBytes = (int) (read - bytesRemaining); + if (extraBytes > 0) { + reportBytesRead -= extraBytes; + delegate.push(b, (int) bytesRemaining, extraBytes); + } + + bytesRemaining -= reportBytesRead; + } + + return reportBytesRead; + } + + @Override + public int read() throws IOException { + var read = read(b1); + if (read <= 0) { + return read; + } + + return b1[0] & 0xFF; + } +} diff --git a/src/main/java/io/fusionauth/http/server/Configurable.java b/src/main/java/io/fusionauth/http/server/Configurable.java index b58c9b0..8c8b694 100644 --- a/src/main/java/io/fusionauth/http/server/Configurable.java +++ b/src/main/java/io/fusionauth/http/server/Configurable.java @@ -62,6 +62,15 @@ default T withChunkedBufferSize(int chunkedBufferSize) { /** * Sets the default compression behavior for the HTTP response. This behavior can be optionally set per response. See * {@link HTTPResponse#setCompress(boolean)}. Defaults to true. + *

+ * Set this configuration to true if you want to compress the response when the Accept-Encoding header is present. Set this + * configuration to false if you want to require the request handler to use {@link HTTPResponse#setCompress(boolean)} in + * order to compress the response. + *

+ * Regardless of this configuration, you always have the option to use {@link HTTPResponse#setCompress(boolean)} on a per-response basis + * as an override. + *

+ * When the request does not contain an Accept-Encoding the response will not be compressed regardless of this configuration. * * @param compressByDefault true if you want to compress by default, or false to not compress by default. * @return This. diff --git a/src/main/java/io/fusionauth/http/server/HTTPRequest.java b/src/main/java/io/fusionauth/http/server/HTTPRequest.java index be5651a..da5e19e 100644 --- a/src/main/java/io/fusionauth/http/server/HTTPRequest.java +++ b/src/main/java/io/fusionauth/http/server/HTTPRequest.java @@ -44,6 +44,7 @@ import io.fusionauth.http.FileInfo; import io.fusionauth.http.HTTPMethod; import io.fusionauth.http.HTTPValues.Connections; +import io.fusionauth.http.HTTPValues.ContentEncodings; import io.fusionauth.http.HTTPValues.ContentTypes; import io.fusionauth.http.HTTPValues.Headers; import io.fusionauth.http.HTTPValues.Protocols; @@ -68,6 +69,8 @@ public class HTTPRequest implements Buildable { private final Map attributes = new HashMap<>(); + private final List contentEncodings = new LinkedList<>(); + private final Map cookies = new HashMap<>(); private final List files = new LinkedList<>(); @@ -147,6 +150,14 @@ public void addAcceptEncodings(List encodings) { this.acceptEncodings.addAll(encodings); } + public void addContentEncoding(String encoding) { + this.contentEncodings.add(encoding); + } + + public void addContentEncodings(List encodings) { + this.contentEncodings.addAll(encodings); + } + public void addCookies(Cookie... cookies) { for (Cookie cookie : cookies) { this.cookies.put(cookie.name, cookie); @@ -296,6 +307,15 @@ public void setCharacterEncoding(Charset encoding) { this.encoding = encoding; } + public List getContentEncodings() { + return contentEncodings; + } + + public void setContentEncodings(List encodings) { + this.contentEncodings.clear(); + this.contentEncodings.addAll(encodings); + } + public Long getContentLength() { return contentLength; } @@ -606,12 +626,16 @@ public void setURLParameters(Map> parameters) { * {@code Content-Length} header was provided. */ public boolean hasBody() { + if (isChunked()) { + return true; + } + Long contentLength = getContentLength(); - return isChunked() || (contentLength != null && contentLength > 0); + return contentLength != null && contentLength > 0; } public boolean isChunked() { - return getTransferEncoding() != null && getTransferEncoding().equalsIgnoreCase(TransferEncodings.Chunked); + return TransferEncodings.Chunked.equalsIgnoreCase(getTransferEncoding()); } /** @@ -756,6 +780,25 @@ private void decodeHeader(String name, String value) { // Ignore the exception and keep the value null } break; + case Headers.ContentEncodingLower: + String[] encodings = value.split(","); + List contentEncodings = new ArrayList<>(1); + for (String encoding : encodings) { + encoding = encoding.trim(); + if (encoding.isEmpty()) { + continue; + } + + // The HTTP/1.1 standard recommends that the servers supporting gzip also recognize x-gzip as an alias for compatibility. + if (encoding.equalsIgnoreCase(ContentEncodings.XGzip)) { + encoding = ContentEncodings.Gzip; + } + + contentEncodings.add(encoding); + } + + setContentEncodings(contentEncodings); + break; case Headers.ContentTypeLower: this.encoding = null; this.multipart = false; diff --git a/src/main/java/io/fusionauth/http/server/internal/HTTPWorker.java b/src/main/java/io/fusionauth/http/server/internal/HTTPWorker.java index 1ae36ff..b77e1da 100644 --- a/src/main/java/io/fusionauth/http/server/internal/HTTPWorker.java +++ b/src/main/java/io/fusionauth/http/server/internal/HTTPWorker.java @@ -25,6 +25,7 @@ import io.fusionauth.http.HTTPProcessingException; import io.fusionauth.http.HTTPValues; import io.fusionauth.http.HTTPValues.Connections; +import io.fusionauth.http.HTTPValues.ContentEncodings; import io.fusionauth.http.HTTPValues.Headers; import io.fusionauth.http.HTTPValues.Protocols; import io.fusionauth.http.ParseException; @@ -417,7 +418,6 @@ private Integer validatePreamble(HTTPRequest request) { // However, as long as we ignore Content-Length we should be ok. Earlier specs indicate Transfer-Encoding should take precedence, // later specs imply it is an error. Seems ok to allow it and just ignore it. if (request.getHeader(Headers.TransferEncoding) == null) { - var contentLength = request.getContentLength(); var requestedContentLengthHeaders = request.getHeaders(Headers.ContentLength); if (requestedContentLengthHeaders != null) { if (requestedContentLengthHeaders.size() != 1) { @@ -429,6 +429,7 @@ private Integer validatePreamble(HTTPRequest request) { return Status.BadRequest; } + var contentLength = request.getContentLength(); if (contentLength == null || contentLength < 0) { if (debugEnabled) { logger.debug("Invalid request. The Content-Length must be >= 0 and <= 9,223,372,036,854,775,807. [{}]", requestedContentLengthHeaders.getFirst()); @@ -444,6 +445,19 @@ private Integer validatePreamble(HTTPRequest request) { request.removeHeader(Headers.ContentLength); } + // Validate Content-Encoding, we currently support deflate and gzip. + // - If we see anything else we should fail, we will be unable to handle the request. + var contentEncodings = request.getContentEncodings(); + for (var encoding : contentEncodings) { + if (!encoding.equalsIgnoreCase(ContentEncodings.Gzip) && !encoding.equalsIgnoreCase(ContentEncodings.Deflate)) { + // Note that while we do not expect multiple Content-Encoding headers, the last one will be used. For good measure, + // use the last one in the debug message as well. + var contentEncodingHeader = request.getHeaders(Headers.ContentEncoding).getLast(); + logger.debug("Invalid request. The Content-Type header contains an un-supported value. [{}]", contentEncodingHeader); + return Status.UnsupportedMediaType; + } + } + return null; } @@ -465,5 +479,7 @@ private static class Status { public static final int HTTPVersionNotSupported = 505; public static final int InternalServerError = 500; + + public static final int UnsupportedMediaType = 415; } } diff --git a/src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java b/src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java index eeb78f3..c7d58d4 100644 --- a/src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java +++ b/src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java @@ -17,9 +17,13 @@ import java.io.IOException; import java.io.InputStream; +import java.util.zip.GZIPInputStream; +import java.util.zip.InflaterInputStream; import io.fusionauth.http.ContentTooLargeException; +import io.fusionauth.http.HTTPValues.ContentEncodings; import io.fusionauth.http.io.ChunkedInputStream; +import io.fusionauth.http.io.FixedLengthInputStream; import io.fusionauth.http.io.PushbackInputStream; import io.fusionauth.http.log.Logger; import io.fusionauth.http.server.HTTPRequest; @@ -27,9 +31,9 @@ import io.fusionauth.http.server.Instrumenter; /** - * An InputStream that handles the HTTP body, including body bytes that were read while the preamble was processed. This class also handles - * chunked bodies by using a delegate InputStream that wraps the original source of the body bytes. The {@link ChunkedInputStream} is the - * delegate that this class leverages for chunking. + * An InputStream intended to read the HTTP request body. + *

+ * This will handle fixed length requests, chunked requests as well as decompression if necessary. * * @author Brian Pontarelli */ @@ -52,16 +56,14 @@ public class HTTPInputStream extends InputStream { private int bytesRead; - private long bytesRemaining; - private boolean closed; - private boolean committed; - private InputStream delegate; private boolean drained; + private boolean initialized; + public HTTPInputStream(HTTPServerConfiguration configuration, HTTPRequest request, PushbackInputStream pushbackInputStream, int maximumContentLength) { this.logger = configuration.getLoggerFactory().getLogger(HTTPInputStream.class); @@ -72,11 +74,6 @@ public HTTPInputStream(HTTPServerConfiguration configuration, HTTPRequest reques this.chunkedBufferSize = configuration.getChunkedBufferSize(); this.maximumBytesToDrain = configuration.getMaxBytesToDrain(); this.maximumContentLength = maximumContentLength; - - // Start the countdown - if (request.getContentLength() != null) { - this.bytesRemaining = request.getContentLength(); - } } @Override @@ -139,73 +136,73 @@ public int read(byte[] b, int off, int len) throws IOException { return 0; } - // If this is a fixed length request, and we have less than or equal to 0 bytes remaining, return -1 - boolean fixedLength = !request.isChunked(); - if (fixedLength && bytesRemaining <= 0) { - return -1; + if (!initialized) { + initialize(); } - if (!committed) { - commit(); - } - - // When we have a fixed length request, read beyond the remainingBytes if possible. - // - If we have read past the end of the current request, push those bytes back onto the InputStream. - // - When a maximum content length has been specified, read at most one byte past the maximum. + // When a maximum content length has been specified, read at most one byte past the maximum. int maxReadLen = maximumContentLength == -1 ? len : Math.min(len, maximumContentLength - bytesRead + 1); int read = delegate.read(b, off, maxReadLen); - - int reportBytesRead = read; - if (fixedLength && read > 0) { - int extraBytes = (int) (read - bytesRemaining); - if (extraBytes > 0) { - reportBytesRead -= extraBytes; - pushbackInputStream.push(b, (int) bytesRemaining, extraBytes); - } - - bytesRemaining -= reportBytesRead; + if (read > 0) { + bytesRead += read; } - bytesRead += reportBytesRead; - - // Note that when the request is fixed length, we will have failed early during commit(). - // - This will handle all requests that are not fixed length. + // Throw an exception once we have read past the maximum configured content length, if (maximumContentLength != -1 && bytesRead > maximumContentLength) { String detailedMessage = "The maximum request size has been exceeded. The maximum request size is [" + maximumContentLength + "] bytes."; throw new ContentTooLargeException(maximumContentLength, detailedMessage); } - return reportBytesRead; + return read; } - private void commit() { - committed = true; + private void initialize() throws IOException { + initialized = true; + + // hasBody means we are either using chunked transfer encoding or we have a non-zero Content-Length. + boolean hasBody = request.hasBody(); + if (hasBody) { + Long contentLength = request.getContentLength(); + // Transfer-Encoding always takes precedence over Content-Length. In practice if they were to both be present on + // the request we would have removed Content-Length during validation to remove ambiguity. See HTTPWorker.validatePreamble. + if (request.isChunked()) { + logger.trace("Client indicated it was sending an entity-body in the request. Handling body using chunked encoding."); + delegate = new ChunkedInputStream(pushbackInputStream, chunkedBufferSize); + if (instrumenter != null) { + instrumenter.chunkedRequest(); + } + } else { + logger.trace("Client indicated it was sending an entity-body in the request. Handling body using Content-Length header {}.", contentLength); + delegate = new FixedLengthInputStream(pushbackInputStream, contentLength); + } - // Note that isChunked() should take precedence over the fact that we have a Content-Length. - // - The client should not send both, but in the case they are both present we ignore Content-Length - // In practice, we will remove the Content-Length header when sent in addition to Transfer-Encoding. See HTTPWorker.validatePreamble. - Long contentLength = request.getContentLength(); - boolean hasBody = (contentLength != null && contentLength > 0) || request.isChunked(); - if (!hasBody) { - delegate = InputStream.nullInputStream(); - } else if (request.isChunked()) { - logger.trace("Client indicated it was sending an entity-body in the request. Handling body using chunked encoding."); - delegate = new ChunkedInputStream(pushbackInputStream, chunkedBufferSize); - if (instrumenter != null) { - instrumenter.chunkedRequest(); + // Now that we have the InputStream set up to read the body, handle decompression. + // The request may contain more than one value, apply in reverse order. + // - These are both using the default 512 buffer size. + for (String contentEncoding : request.getContentEncodings().reversed()) { + if (contentEncoding.equalsIgnoreCase(ContentEncodings.Deflate)) { + delegate = new InflaterInputStream(delegate); + } else if (contentEncoding.equalsIgnoreCase(ContentEncodings.Gzip)) { + delegate = new GZIPInputStream(delegate); + } + } + + // If we have a fixed length request that is reporting a contentLength larger than the configured maximum, fail early. + // - Do this last so if anyone downstream wants to read from the InputStream it would work. + // - Note that it is possible that the body is compressed which would mean the contentLength represents the compressed value. + // But when we decompress the bytes the result will be larger than the reported contentLength, so we can safely throw this exception. + if (contentLength != null && maximumContentLength != -1 && contentLength > maximumContentLength) { + String detailedMessage = "The maximum request size has been exceeded. The reported Content-Length is [" + contentLength + "] and the maximum request size is [" + maximumContentLength + "] bytes."; + throw new ContentTooLargeException(maximumContentLength, detailedMessage); } - } else if (contentLength != null) { - logger.trace("Client indicated it was sending an entity-body in the request. Handling body using Content-Length header {}.", contentLength); } else { + // This means that we did not find Content-Length or Transfer-Encoding on the request. Do not attempt to read from the InputStream. + // - Note that the spec indicates it is plausible for a client to send an entity body and omit these two headers and the server can optionally + // read bytes until the end of the InputStream is reached. This would assume Connection: close was also sent because if we do not know + // how to delimit the request we cannot use a persistent connection. + // - We aren't doing any of that - if the client wants to send bytes, it needs to send a Content-Length header, or specify Transfer-Encoding: chunked. logger.trace("Client indicated it was NOT sending an entity-body in the request"); - } - - // If we have a maximumContentLength, and this is a fixed content length request, before we read any bytes, fail early. - // For good measure do this last so if anyone downstream wants to read from the InputStream they could in theory because - // we will have set up the InputStream. - if (contentLength != null && maximumContentLength != -1 && contentLength > maximumContentLength) { - String detailedMessage = "The maximum request size has been exceeded. The reported Content-Length is [" + contentLength + "] and the maximum request size is [" + maximumContentLength + "] bytes."; - throw new ContentTooLargeException(maximumContentLength, detailedMessage); + delegate = InputStream.nullInputStream(); } } } diff --git a/src/main/java/io/fusionauth/http/server/io/HTTPOutputStream.java b/src/main/java/io/fusionauth/http/server/io/HTTPOutputStream.java index 7a29a32..805df9b 100644 --- a/src/main/java/io/fusionauth/http/server/io/HTTPOutputStream.java +++ b/src/main/java/io/fusionauth/http/server/io/HTTPOutputStream.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024, FusionAuth, All Rights Reserved + * Copyright (c) 2024-2025, FusionAuth, All Rights Reserved * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/src/test/java/io/fusionauth/http/BaseSocketTest.java b/src/test/java/io/fusionauth/http/BaseSocketTest.java index 5150d29..1a57900 100644 --- a/src/test/java/io/fusionauth/http/BaseSocketTest.java +++ b/src/test/java/io/fusionauth/http/BaseSocketTest.java @@ -66,7 +66,8 @@ private void assertResponse(String request, String chunkedExtension, String resp var body = bodyString.repeat(((requestBufferSize / bodyString.length())) * 2); if (request.contains("Transfer-Encoding: chunked")) { - body = chunkItUp(body, chunkedExtension); + // Chunk in 100 byte increments. Using a smaller chunk size to ensure we don't end up with a single chunk. + body = new String(chunkEncode(body.getBytes(StandardCharsets.UTF_8), 100, chunkedExtension)); } request = request.replace("{body}", body); diff --git a/src/test/java/io/fusionauth/http/BaseTest.java b/src/test/java/io/fusionauth/http/BaseTest.java index bb7b390..93eb292 100644 --- a/src/test/java/io/fusionauth/http/BaseTest.java +++ b/src/test/java/io/fusionauth/http/BaseTest.java @@ -15,6 +15,8 @@ */ package io.fusionauth.http; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -47,15 +49,18 @@ import java.time.Instant; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; -import java.util.ArrayList; import java.util.Arrays; import java.util.Date; -import java.util.List; import java.util.Map; import java.util.UUID; import java.util.stream.Collectors; +import java.util.zip.DeflaterOutputStream; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; +import java.util.zip.InflaterInputStream; import io.fusionauth.http.HTTPValues.Connections; +import io.fusionauth.http.HTTPValues.ContentEncodings; import io.fusionauth.http.log.FileLogger; import io.fusionauth.http.log.FileLoggerFactory; import io.fusionauth.http.log.Level; @@ -100,7 +105,6 @@ * @author Brian Pontarelli */ public abstract class BaseTest { - /** * This timeout is used for the HttpClient during each test. If you are in a debugger, you will need to change this timeout to be much * larger, otherwise, the client might truncate the request to the server. @@ -244,6 +248,17 @@ public Object[][] connections() { }; } + @DataProvider(name = "contentEncoding") + public Object[][] contentEncoding() { + return new Object[][]{ + {""}, // No compression + {"gzip"}, // gzip only + {"deflate"}, // deflate only + {"gzip, deflate"}, // gzip, then deflate + {"deflate, gzip"} // deflate, then gzip + }; + } + @AfterMethod public void flush() { FileLogger fl = (FileLogger) FileLoggerFactory.FACTORY.getLogger(BaseTest.class); @@ -430,27 +445,71 @@ protected void assertHTTPResponseEquals(Socket socket, String expectedResponse) } } - protected String chunkItUp(String body, String chunkedExtension) { - List result = new ArrayList<>(); - // Chunk in 100 byte increments. Using a smaller chunk size to ensure we don't end up with a single chunk. - int chunkSize = 100; - for (var i = 0; i < body.length(); i += chunkSize) { - var endIndex = Math.min(i + chunkSize, body.length()); - var chunk = body.substring(i, endIndex); - var chunkLength = chunk.getBytes(StandardCharsets.UTF_8).length; + protected byte[] chunkEncode(byte[] bytes, int chunkSize, String chunkedExtension) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + for (var i = 0; i < bytes.length; i += chunkSize) { + var endIndex = Math.min(i + chunkSize, bytes.length); + + var chunk = Arrays.copyOfRange(bytes, i, endIndex); + var chunkLength = chunk.length; String hex = Integer.toHexString(chunkLength); - result.add(hex); + out.write(hex.getBytes(StandardCharsets.UTF_8)); if (chunkedExtension != null) { - result.add(chunkedExtension); + out.write(chunkedExtension.getBytes(StandardCharsets.UTF_8)); + } + + out.write("\r\n".getBytes(StandardCharsets.UTF_8)); + out.write(chunk); + out.write("\r\n".getBytes(StandardCharsets.UTF_8)); + } + + out.write(("0\r\n\r\n".getBytes(StandardCharsets.UTF_8))); + return out.toByteArray(); + } + + protected byte[] compressUsingContentEncoding(byte[] bytes, String contentEncoding) throws Exception { + if (!contentEncoding.isEmpty()) { + var requestEncodings = contentEncoding.toLowerCase().trim().split(","); + for (String part : requestEncodings) { + String encoding = part.trim(); + if (encoding.equals(ContentEncodings.Deflate)) { + bytes = deflate(bytes); + } else if (encoding.equals(ContentEncodings.Gzip)) { + bytes = gzip(bytes); + } } + } + + return bytes; + } + + protected byte[] deflate(byte[] bytes) throws Exception { + ByteArrayOutputStream baseOutputStream = new ByteArrayOutputStream(); + try (DeflaterOutputStream out = new DeflaterOutputStream(baseOutputStream, true)) { + out.write(bytes); + out.flush(); + out.finish(); + return baseOutputStream.toByteArray(); + } + } - result.add(("\r\n" + chunk + "\r\n")); + protected byte[] gzip(byte[] bytes) throws Exception { + ByteArrayOutputStream baseOutputStream = new ByteArrayOutputStream(); + try (DeflaterOutputStream out = new GZIPOutputStream(baseOutputStream, true)) { + out.write(bytes); + out.flush(); + out.finish(); + return baseOutputStream.toByteArray(); } + } - result.add(("0\r\n\r\n")); - return String.join("", result); + protected byte[] inflate(byte[] bytes) throws Exception { + ByteArrayInputStream baseInputStream = new ByteArrayInputStream(bytes); + try (InflaterInputStream in = new InflaterInputStream(baseInputStream)) { + return in.readAllBytes(); + } } protected void printf(String format, Object... args) { @@ -472,6 +531,13 @@ protected void sleep(long millis) { } } + protected byte[] ungzip(byte[] bytes) throws Exception { + ByteArrayInputStream baseInputStream = new ByteArrayInputStream(bytes); + try (InflaterInputStream in = new GZIPInputStream(baseInputStream)) { + return in.readAllBytes(); + } + } + /** * Verifies that the chain certificates can be validated up to the supplied root certificate. See * {@link CertPathValidator#validate(CertPath, CertPathParameters)} for details. diff --git a/src/test/java/io/fusionauth/http/ChunkedTest.java b/src/test/java/io/fusionauth/http/ChunkedTest.java index 4a9d297..0639b4e 100644 --- a/src/test/java/io/fusionauth/http/ChunkedTest.java +++ b/src/test/java/io/fusionauth/http/ChunkedTest.java @@ -144,7 +144,7 @@ public void chunkedRequest_doNotReadTheInputStream(String scheme) throws Excepti var response = client.send(HttpRequest.newBuilder() .uri(uri) .header(Headers.ContentType, "text/plain") - // Note that using a InputStream baed publisher will caues the JDK to + // Note that using a InputStream based publisher will caues the JDK to // enable Transfer-Encoding: chunked .POST(BodyPublishers.ofInputStream(() -> new ByteArrayInputStream(responseBodyBytes))) diff --git a/src/test/java/io/fusionauth/http/CompressionTest.java b/src/test/java/io/fusionauth/http/CompressionTest.java index 045daf9..a866013 100644 --- a/src/test/java/io/fusionauth/http/CompressionTest.java +++ b/src/test/java/io/fusionauth/http/CompressionTest.java @@ -15,12 +15,14 @@ */ package io.fusionauth.http; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.math.BigDecimal; import java.net.URI; import java.net.http.HttpRequest; +import java.net.http.HttpRequest.BodyPublishers; import java.net.http.HttpResponse.BodySubscribers; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -38,6 +40,8 @@ import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotEquals; +import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; import static org.testng.AssertJUnit.fail; @@ -52,63 +56,109 @@ public class CompressionTest extends BaseTest { @DataProvider(name = "chunkedSchemes") public Object[][] chunkedSchemes() { return new Object[][]{ - {true, "http"}, - {true, "https"}, - {false, "http"}, - {false, "https"} + {"http", true}, + {"http", false}, + {"https", true}, + {"https", false} }; } @Test(dataProvider = "compressedChunkedSchemes") - public void compress(String encoding, boolean chunked, String scheme) throws Exception { + public void compress(String scheme, String contentEncoding, String acceptEncoding) throws Exception { HTTPHandler handler = (req, res) -> { - // Testing an indecisive user, can't make up their mind... this is allowed as long as you have not written ay bytes. - res.setCompress(true); - res.setCompress(false); - res.setCompress(true); - res.setCompress(false); - res.setCompress(true); + + String body = new String(req.getInputStream().readAllBytes()); + assertEquals(body, "Hello world!"); res.setHeader(Headers.ContentType, "text/plain"); res.setStatus(200); // Technically, this is ignored anytime compression is used, but we are testing if folks don't set it here - if (!chunked) { - res.setContentLength(Files.size(file)); - } + res.setContentLength(Files.size(file)); try (InputStream is = Files.newInputStream(file)) { OutputStream outputStream = res.getOutputStream(); is.transferTo(outputStream); - - // Try to call setCompress, expect an exception - we cannot change modes once we've written to the OutputStream. - try { - res.setCompress(false); - fail("Expected setCompress(false) to fail hard!"); - } catch (IllegalStateException expected) { - } - outputStream.close(); } catch (IOException e) { throw new RuntimeException(e); } }; + String bodyString = "Hello world!"; + byte[] bodyBytes = bodyString.getBytes(StandardCharsets.UTF_8); + var payload = bodyBytes; + + if (!contentEncoding.isEmpty()) { + var requestEncodings = contentEncoding.toLowerCase().trim().split(","); + for (String part : requestEncodings) { + String encoding = part.trim(); + if (encoding.equals(ContentEncodings.Deflate)) { + payload = deflate(payload); + } else if (encoding.equals(ContentEncodings.Gzip) || encoding.equals(ContentEncodings.XGzip)) { + payload = gzip(payload); + } + } + + byte[] uncompressedBody = payload; + + // Sanity check on round trip compress/decompress + for (int i = requestEncodings.length - 1; i >= 0; i--) { + String encoding = requestEncodings[i].trim(); + if (encoding.equals(ContentEncodings.Deflate)) { + uncompressedBody = inflate(uncompressedBody); + } else if (encoding.equals(ContentEncodings.Gzip) || encoding.equals(ContentEncodings.XGzip)) { + uncompressedBody = ungzip(uncompressedBody); + } + } + + assertEquals(uncompressedBody, bodyBytes); + assertEquals(new String(bodyBytes), bodyString); + } + + var requestPayload = payload; + + // Make the request CountingInstrumenter instrumenter = new CountingInstrumenter(); try (var client = makeClient(scheme, null); var ignore = makeServer(scheme, handler, instrumenter).start()) { URI uri = makeURI(scheme, ""); var response = client.send( - HttpRequest.newBuilder().header(Headers.AcceptEncoding, encoding).uri(uri).GET().build(), + HttpRequest.newBuilder() + .uri(uri) + .header(Headers.AcceptEncoding, acceptEncoding) + .header(Headers.ContentEncoding, contentEncoding) + .header(Headers.ContentType, "text/plain") + // In general using a BodyPublishers.ofInputStream causes the client to use chunked transfer encoding. + // - Manually set the header because the body is small, and it may not chunk it otherwise. + .header(Headers.TransferEncoding, "chunked") + .POST(BodyPublishers.ofInputStream(() -> new ByteArrayInputStream(requestPayload))) + .build(), r -> BodySubscribers.ofInputStream() ); - var result = new String( - encoding.equals(ContentEncodings.Deflate) - ? new InflaterInputStream(response.body()).readAllBytes() - : new GZIPInputStream(response.body()).readAllBytes(), StandardCharsets.UTF_8); - - assertEquals(response.headers().firstValue(Headers.ContentEncoding).orElse(null), encoding); assertEquals(response.statusCode(), 200); + String expectedResponseEncoding = null; + for (String part : acceptEncoding.toLowerCase().trim().split(",")) { + expectedResponseEncoding = part.trim(); + break; + } + + assertNotNull(expectedResponseEncoding); + + String result = null; + InputStream responseInputStream = response.body(); + + if (expectedResponseEncoding.isEmpty()) { + result = new String(responseInputStream.readAllBytes()); + } else { + assertEquals(response.headers().firstValue(Headers.ContentEncoding).orElse(null), expectedResponseEncoding); + if (expectedResponseEncoding.equals(ContentEncodings.Deflate)) { + result = new String(new InflaterInputStream(responseInputStream).readAllBytes()); + } else if (expectedResponseEncoding.equals(ContentEncodings.Gzip) || expectedResponseEncoding.equals(ContentEncodings.XGzip)) { + result = new String(new GZIPInputStream(responseInputStream).readAllBytes(), StandardCharsets.UTF_8); + } + } + assertEquals(result, Files.readString(file)); } } @@ -197,71 +247,115 @@ public void compressPerformance() throws Exception { } } - @Test(dataProvider = "compressedChunkedSchemes") - public void compress_onByDefault(String encoding, boolean chunked, String scheme) throws Exception { + @Test(dataProvider = "schemes") + public void compressWithContentLength(String scheme) throws Exception { + // Use case: Compress the request w/out using chunked transfer encoding. + // - The JDK rest client is hard to predict sometimes, but I think the body is small enough it won't chunk it up. + String bodyString = "Hello world!"; + byte[] bodyBytes = bodyString.getBytes(StandardCharsets.UTF_8); + var payload = gzip(bodyBytes); + HTTPHandler handler = (req, res) -> { - // Use case, do not call response.setCompress(true) + + assertEquals(req.isChunked(), false); + + // We forced a Content-Length by telling the JDK client not to chunk, so it will be present. + // - We can still correctly use the Content-Length because it is used by the FixedLengthInputStream which + // is below the decompression. See HTTPInputStream.initialize + var contentLength = req.getHeader(Headers.ContentLength); + assertEquals(contentLength, payload.length + ""); + + // Because we are compressed, don't expect the final payload to be equal to the contentLength + byte[] readBytes = req.getInputStream().readAllBytes(); + assertNotEquals(readBytes.length, contentLength); + + String body = new String(readBytes); + assertEquals(body, bodyString); + res.setHeader(Headers.ContentType, "text/plain"); res.setStatus(200); + }; - // Technically, this is ignored anytime compression is used, but we are testing if folks don't set it here - if (!chunked) { - res.setContentLength(Files.size(file)); - } + // This publisher should cause a fixed length request. + var bodyPublisher = BodyPublishers.ofByteArray(payload); - try (InputStream is = Files.newInputStream(file)) { - OutputStream outputStream = res.getOutputStream(); - is.transferTo(outputStream); - outputStream.close(); - } catch (IOException e) { - throw new RuntimeException(e); - } - }; + try (var client = makeClient(scheme, null); + var ignore = makeServer(scheme, handler, null).start()) { - CountingInstrumenter instrumenter = new CountingInstrumenter(); - try (var client = makeClient(scheme, null); var ignore = makeServer(scheme, handler, instrumenter).start()) { URI uri = makeURI(scheme, ""); var response = client.send( - HttpRequest.newBuilder().header(Headers.AcceptEncoding, encoding).uri(uri).GET().build(), - r -> BodySubscribers.ofInputStream() + HttpRequest.newBuilder() + .uri(uri) + .header(Headers.ContentEncoding, "gzip") + .header(Headers.ContentType, "text/plain") + .POST(bodyPublisher) + .build(), + r -> BodySubscribers.ofString(StandardCharsets.UTF_8) ); - var result = new String( - encoding.equals(ContentEncodings.Deflate) - ? new InflaterInputStream(response.body()).readAllBytes() - : new GZIPInputStream(response.body()).readAllBytes(), StandardCharsets.UTF_8); - - assertEquals(response.headers().firstValue(Headers.ContentEncoding).orElse(null), encoding); assertEquals(response.statusCode(), 200); - assertEquals(result, Files.readString(file)); + assertEquals(response.body(), ""); } } @DataProvider(name = "compressedChunkedSchemes") public Object[][] compressedChunkedSchemes() { - return new Object[][]{ - // encoding, chunked, schema + Object[][] compressOptions = new Object[][]{ + // Content-Encoding, Accept-Encoding + + // No compression on request, or response + {"", ""}, - // Chunked http - {ContentEncodings.Deflate, true, "http"}, - {ContentEncodings.Gzip, true, "http"}, + // Only request + {"deflate", ""}, + {"gzip", ""}, - // Chunked https - {ContentEncodings.Deflate, true, "https"}, - {ContentEncodings.Gzip, true, "https"}, + // Only response + {"", "deflate"}, + {"", "gzip"}, - // Non chunked http - {ContentEncodings.Deflate, false, "http"}, - {ContentEncodings.Gzip, false, "http"}, + // Same on request and response + {"deflate", "deflate"}, + {"gzip", "gzip"}, - // Non chunked https - {ContentEncodings.Deflate, false, "https"}, - {ContentEncodings.Gzip, false, "https"} + // UC, and mixed case, http + {"Deflate", "Deflate"}, + {"Gzip", "Gzip"}, + + // Multiple accept values, expect to use the first, http + {"deflate", "deflate, gzip"}, + {"gzip", "gzip, deflate"}, + + // Multiple request values, this means we will use multiple passes of compression, https + {"deflate, gzip", "deflate, gzip"}, + {"gzip, deflate", "gzip, deflate"}, + + // x-gzip with mixed case alias, https + {"x-gzip", "deflate, gzip"}, + {"X-Gzip, deflate", "gzip, deflate"}, + {"deflate, x-gzip", "gzip, deflate"}, }; + + // For each scheme + Object[][] schemes = schemes(); + Object[][] result = new Object[compressOptions.length * 2][3]; + + int index = 0; + for (Object[] compressOption : compressOptions) { + for (Object[] scheme : schemes) { + result[index][0] = scheme[0]; // scheme + result[index][1] = compressOption[0]; // Content-Encoding + result[index][2] = compressOption[1]; // Accept-Encoding + index++; + } + } + + // scheme, Content-Encoding, Accept-Encoding + return result; } @Test(dataProvider = "chunkedSchemes") - public void requestedButNotAccepted(boolean chunked, String scheme) throws Exception { + public void requestedButNotAccepted(String scheme, boolean chunked) throws Exception { // Use case: setCompress(true), but the request does not contain the 'Accept-Encoding' header. // Result: no compression HTTPHandler handler = (req, res) -> { @@ -298,11 +392,19 @@ public void requestedButNotAccepted(boolean chunked, String scheme) throws Excep } @Test(dataProvider = "chunkedSchemes") - public void requestedButNotAccepted_unSupportedEncoding(boolean chunked, String scheme) throws Exception { + public void requestedButNotAccepted_unSupportedEncoding(String scheme, boolean chunked) throws Exception { // Use case: setCompress(true), and 'Accept-Encoding: br' which is valid, but not yet supported // Result: no compression HTTPHandler handler = (req, res) -> { res.setCompress(true); + + // Testing an indecisive user, can't make up their mind... this is allowed as long as you have not written ay bytes. + res.setCompress(true); + res.setCompress(false); + res.setCompress(true); + res.setCompress(false); + res.setCompress(true); + res.setHeader(Headers.ContentType, "text/plain"); res.setStatus(200); @@ -314,6 +416,14 @@ public void requestedButNotAccepted_unSupportedEncoding(boolean chunked, String try (InputStream is = Files.newInputStream(file)) { OutputStream outputStream = res.getOutputStream(); is.transferTo(outputStream); + + // Try to call setCompress, expect an exception - we cannot change modes once we've written to the OutputStream. + try { + res.setCompress(false); + fail("Expected setCompress(false) to fail hard!"); + } catch (IllegalStateException expected) { + } + outputStream.close(); } catch (IOException e) { throw new RuntimeException(e); @@ -333,4 +443,35 @@ public void requestedButNotAccepted_unSupportedEncoding(boolean chunked, String assertEquals(response.body(), Files.readString(file)); } } + + @Test(dataProvider = "schemes") + public void unsupportedContentType(String scheme) throws Exception { + // Use case: Tell the server we encoded the request using 'br' + HTTPHandler handler = (req, res) -> { + res.setHeader(Headers.ContentType, "text/plain"); + res.setStatus(200); + }; + + // The body isn't actually encoded, that is ok, we will validate that 'br' is not supported when we validate the preamble. + var bodyPublisher = BodyPublishers.ofInputStream(() -> new ByteArrayInputStream("Hello World".getBytes(StandardCharsets.UTF_8))); + + try (var client = makeClient(scheme, null); + var ignore = makeServer(scheme, handler, null).start()) { + + URI uri = makeURI(scheme, ""); + var response = client.send( + HttpRequest.newBuilder() + .uri(uri) + .header(Headers.ContentEncoding, "br") + .header(Headers.ContentType, "text/plain") + .POST(bodyPublisher) + .build(), + r -> BodySubscribers.ofString(StandardCharsets.UTF_8) + ); + + // Expect a 415 w/ an empty body response + assertEquals(response.statusCode(), 415); + assertEquals(response.body(), ""); + } + } } diff --git a/src/test/java/io/fusionauth/http/FormDataTest.java b/src/test/java/io/fusionauth/http/FormDataTest.java index f6506f9..88f21e2 100644 --- a/src/test/java/io/fusionauth/http/FormDataTest.java +++ b/src/test/java/io/fusionauth/http/FormDataTest.java @@ -199,6 +199,17 @@ public Builder(String scheme) { this.scheme = scheme; } + public Builder assertOptionalExceptionOnWrite(Class clazz) { + // Note that this assertion really depends upon the system the test is run on, the size of the request, and the amount of data that can be cached. + // - So this is an optional assertion - if exception is not null, then we should be able to assert some attributes. + // - With the larger sizes this exception is mostly always thrown when running tests locally, but in GHA, it doesn't always occur. + if (thrownOnWrite != null) { + assertEquals(thrownOnWrite.getClass(), clazz); + } + + return this; + } + public Builder expectNoExceptionOnWrite() { assertNull(thrownOnWrite); return this; @@ -274,7 +285,8 @@ public Builder expectResponse(String response) throws Exception { """; // Convert body to chunked - body = chunkItUp(body, null); + // - Using a small chunk to ensure we end up with more than one chunk. + body = new String(chunkEncode(body.getBytes(StandardCharsets.UTF_8), 100, null)); } else { var contentLength = body.getBytes(StandardCharsets.UTF_8).length; if (contentLength > 0) { @@ -309,17 +321,6 @@ public Builder expectResponse(String response) throws Exception { return this; } - public Builder assertOptionalExceptionOnWrite(Class clazz) { - // Note that this assertion really depends upon the system the test is run on, the size of the request, and the amount of data that can be cached. - // - So this is an optional assertion - if exception is not null, then we should be able to assert some attributes. - // - With the larger sizes this exception is mostly always thrown when running tests locally, but in GHA, it doesn't always occur. - if (thrownOnWrite != null) { - assertEquals(thrownOnWrite.getClass(), clazz); - } - - return this; - } - public Builder withBodyParameterCount(int bodyParameterCount) { this.bodyParameterCount = bodyParameterCount; return this; diff --git a/src/test/java/io/fusionauth/http/MultipartTest.java b/src/test/java/io/fusionauth/http/MultipartTest.java index 3ecc7ff..413d1a0 100644 --- a/src/test/java/io/fusionauth/http/MultipartTest.java +++ b/src/test/java/io/fusionauth/http/MultipartTest.java @@ -39,7 +39,6 @@ import io.fusionauth.http.server.HTTPServerConfiguration; import org.testng.annotations.Test; import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; /** @@ -128,7 +127,7 @@ public void post_server_configuration_fileTooBig(String scheme) throws Exception content-length: 0\r \r """) - .expectExceptionOnWrite(SocketException.class); + .assertOptionalExceptionOnWrite(SocketException.class); } @Test(dataProvider = "schemes") @@ -187,7 +186,7 @@ public void post_server_configuration_file_upload_reject(String scheme) throws E """) // If the request is large enough, because we throw an exception before we have emptied the InputStream // we will take an exception while trying to write all the bytes to the server. - .expectExceptionOnWrite(SocketException.class); + .assertOptionalExceptionOnWrite(SocketException.class); } @@ -212,7 +211,7 @@ public void post_server_configuration_requestTooBig(String scheme) throws Except """) // If the request is large enough, because we throw an exception before we have emptied the InputStream // we will take an exception while trying to write all the bytes to the server. - .expectExceptionOnWrite(SocketException.class); + .assertOptionalExceptionOnWrite(SocketException.class); } @Test(dataProvider = "schemes") @@ -240,7 +239,7 @@ public void post_server_configuration_requestTooBig_maxBodySize(String scheme) t """) // If the request is large enough, because we throw an exception before we have emptied the InputStream // we will take an exception while trying to write all the bytes to the server. - .expectExceptionOnWrite(SocketException.class); + .assertOptionalExceptionOnWrite(SocketException.class); } private Builder withConfiguration(Consumer configuration) throws Exception { @@ -269,9 +268,14 @@ public Builder(String scheme) { this.scheme = scheme; } - public Builder expectExceptionOnWrite(Class clazz) { - assertNotNull(thrownOnWrite); - assertEquals(thrownOnWrite.getClass(), clazz); + public Builder assertOptionalExceptionOnWrite(Class clazz) { + // Note that this assertion really depends upon the system the test is run on, the size of the request, and the amount of data that can be cached. + // - So this is an optional assertion - if exception is not null, then we should be able to assert some attributes. + // - With the larger sizes this exception is mostly always thrown when running tests locally, but in GHA, it doesn't always occur. + if (thrownOnWrite != null) { + assertEquals(thrownOnWrite.getClass(), clazz); + } + return this; } diff --git a/src/test/java/io/fusionauth/http/io/HTTPInputStreamTest.java b/src/test/java/io/fusionauth/http/io/HTTPInputStreamTest.java index 3dab298..bf8f4bf 100644 --- a/src/test/java/io/fusionauth/http/io/HTTPInputStreamTest.java +++ b/src/test/java/io/fusionauth/http/io/HTTPInputStreamTest.java @@ -16,8 +16,11 @@ package io.fusionauth.http.io; import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.nio.charset.StandardCharsets; +import io.fusionauth.http.BaseTest; +import io.fusionauth.http.HTTPValues.Headers; import io.fusionauth.http.server.HTTPRequest; import io.fusionauth.http.server.HTTPServerConfiguration; import io.fusionauth.http.server.io.HTTPInputStream; @@ -27,52 +30,74 @@ /** * @author Daniel DeGroff */ -public class HTTPInputStreamTest { - @Test - public void read_chunked_withPushback() throws Exception { +public class HTTPInputStreamTest extends BaseTest { + @Test(dataProvider = "contentEncoding") + public void read_chunked_withPushback(String contentEncoding) throws Exception { // Ensure that when we read a chunked encoded body that the InputStream returns the correct number of bytes read even when // we read past the end of the current request and use the PushbackInputStream. + // - Test with optional compression as well. String content = "These pretzels are making me thirsty. These pretzels are making me thirsty. These pretzels are making me thirsty."; - int contentLength = content.getBytes(StandardCharsets.UTF_8).length; - - // Chunk the content - byte[] bytes = """ - 26\r - These pretzels are making me thirsty. \r - 26\r - These pretzels are making me thirsty. \r - 25\r - These pretzels are making me thirsty.\r - 0\r - \r - GET / HTTP/1.1\r - """.getBytes(); + byte[] payload = content.getBytes(StandardCharsets.UTF_8); + int contentLength = payload.length; + + // Optionally compress + payload = compressUsingContentEncoding(payload, contentEncoding); + + // Chunk the content, add part of the next request + payload = chunkEncode(payload, 38, null); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + out.write(payload); + + // Add part of the next request + String nextRequest = "GET / HTTP/1.1\n\r"; + byte[] nextRequestBytes = nextRequest.getBytes(StandardCharsets.UTF_8); + out.write(nextRequestBytes); HTTPRequest request = new HTTPRequest(); - request.setHeader("Transfer-Encoding", "chunked"); + request.setHeader(Headers.ContentEncoding, contentEncoding); + request.setHeader(Headers.TransferEncoding, "chunked"); - assertReadWithPushback(bytes, content, contentLength, request); + byte[] bytes = out.toByteArray(); + assertReadWithPushback(bytes, content, contentLength, nextRequestBytes, request); } - @Test - public void read_fixedLength_withPushback() throws Exception { + @Test(dataProvider = "contentEncoding") + public void read_fixedLength_withPushback(String contentEncoding) throws Exception { // Ensure that when we read a fixed length body that the InputStream returns the correct number of bytes read even when // we read past the end of the current request and use the PushbackInputStream. + // - Test with optional compression as well. + // Fixed length body with the start of the next request in the buffer String content = "These pretzels are making me thirsty. These pretzels are making me thirsty. These pretzels are making me thirsty."; - int contentLength = content.getBytes(StandardCharsets.UTF_8).length; + byte[] payload = content.getBytes(StandardCharsets.UTF_8); + int contentLength = payload.length; - // Fixed length body with the start of the next request in the buffer - byte[] bytes = (content + "GET / HTTP/1.1\r\n").getBytes(StandardCharsets.UTF_8); + // Optionally compress + payload = compressUsingContentEncoding(payload, contentEncoding); + + // Content-Length must be the compressed length + int compressedLength = payload.length; + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + out.write(payload); + + // Add part of the next request + String nextRequest = "GET / HTTP/1.1\n\r"; + byte[] nextRequestBytes = nextRequest.getBytes(StandardCharsets.UTF_8); + out.write(nextRequestBytes); HTTPRequest request = new HTTPRequest(); - request.setHeader("Content-Length", contentLength + ""); + request.setHeader(Headers.ContentEncoding, contentEncoding); + request.setHeader(Headers.ContentLength, compressedLength + ""); - assertReadWithPushback(bytes, content, contentLength, request); + byte[] bytes = out.toByteArray(); + assertReadWithPushback(bytes, content, contentLength, nextRequestBytes, request); } - private void assertReadWithPushback(byte[] bytes, String content, int contentLength, HTTPRequest request) throws Exception { + private void assertReadWithPushback(byte[] bytes, String content, int contentLength, byte[] pushedBackBytes, HTTPRequest request) + throws Exception { int bytesAvailable = bytes.length; HTTPServerConfiguration configuration = new HTTPServerConfiguration().withRequestBufferSize(bytesAvailable + 100); @@ -83,6 +108,9 @@ private void assertReadWithPushback(byte[] bytes, String content, int contentLen byte[] buffer = new byte[configuration.getRequestBufferSize()]; int read = httpInputStream.read(buffer); + // Note that the HTTPInputStream read will return the number of uncompressed bytes read. So the contentLength passed in + // needs to represent the actual length of the request body, not the value of the Content-Length sent on the request which + // would represent the size of the compressed entity body. assertEquals(read, contentLength); assertEquals(new String(buffer, 0, read), content); @@ -90,13 +118,13 @@ private void assertReadWithPushback(byte[] bytes, String content, int contentLen int secondRead = httpInputStream.read(buffer); assertEquals(secondRead, -1); - // We have 16 bytes left over - assertEquals(pushbackInputStream.getAvailableBufferedBytesRemaining(), 16); + // Expect that we have the number of bytes left over we expect + assertEquals(pushbackInputStream.getAvailableBufferedBytesRemaining(), pushedBackBytes.length); // Next read should start at the next request byte[] leftOverBuffer = new byte[100]; int leftOverRead = pushbackInputStream.read(leftOverBuffer); - assertEquals(leftOverRead, 16); - assertEquals(new String(leftOverBuffer, 0, leftOverRead), "GET / HTTP/1.1\r\n"); + assertEquals(leftOverRead, pushedBackBytes.length); + assertEquals(new String(leftOverBuffer, 0, leftOverRead), new String(pushedBackBytes)); } } diff --git a/src/test/java/io/fusionauth/http/util/HTTPToolsTest.java b/src/test/java/io/fusionauth/http/util/HTTPToolsTest.java index 50f2e47..bf24f5b 100644 --- a/src/test/java/io/fusionauth/http/util/HTTPToolsTest.java +++ b/src/test/java/io/fusionauth/http/util/HTTPToolsTest.java @@ -16,6 +16,7 @@ package io.fusionauth.http.util; import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.net.URLEncoder; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; @@ -26,6 +27,7 @@ import java.util.Map; import com.inversoft.json.ToString; +import io.fusionauth.http.BaseTest; import io.fusionauth.http.HTTPMethod; import io.fusionauth.http.io.PushbackInputStream; import io.fusionauth.http.server.HTTPRequest; @@ -42,7 +44,7 @@ * @author Brian Pontarelli */ @Test -public class HTTPToolsTest { +public class HTTPToolsTest extends BaseTest { @Test public void getMaxRequestBodySize() { var configuration = Map.of( @@ -183,16 +185,18 @@ public void parseHeaderValue() { assertEquals(HTTPTools.parseHeaderValue("value; f= f"), new HeaderValue("value", Map.of("f", " f"))); } - @Test - public void parsePreamble() throws Exception { + @Test(dataProvider = "contentEncoding") + public void parsePreamble(String contentEncoding) throws Exception { // Ensure that we can correctly read the preamble when the InputStream contains the next request. + // - Optionally compress the body to ensure we can push back bytes on the preamble read w/out hosing it up. - //noinspection ExtractMethodRecommender - String request = """ + ByteArrayOutputStream out = new ByteArrayOutputStream(); + String header = """ GET / HTTP/1.1\r Host: localhost:42\r Connection: close\r - Content-Length: 113\r + Content-Encoding: {contentEncoding}\r + Content-Length: {contentLength}\r Header1: Value1\r Header2: Value2\r Header3: Value3\r @@ -204,17 +208,31 @@ public void parsePreamble() throws Exception { Header9: Value9\r Header10: Value10\r \r - These pretzels are making me thirsty. These pretzels are making me thirsty. These pretzels are making me thirsty.GET / HTTP/1.1\r - """; + """.replace("{contentEncoding}", contentEncoding); + + // Now add the body, optionally compressed + String body = "These pretzels are making me thirsty. These pretzels are making me thirsty. These pretzels are making me thirsty."; + byte[] bodyBytes = body.getBytes(StandardCharsets.UTF_8); + int uncompressedBodyLength = bodyBytes.length; + bodyBytes = compressUsingContentEncoding(bodyBytes, contentEncoding); + + header = header.replace("{contentLength}", bodyBytes.length + ""); + out.write(header.getBytes(StandardCharsets.UTF_8)); + out.write(bodyBytes); + + // Now add part of the next request + String nextRequest = "GET / HTTP/1.1\n\r"; + byte[] nextRequestBytes = nextRequest.getBytes(StandardCharsets.UTF_8); + out.write(nextRequestBytes); // Fixed length body with the start of the next request in the buffer - byte[] bytes = request.getBytes(StandardCharsets.UTF_8); - int bytesAvailable = bytes.length; + byte[] payload = out.toByteArray(); + int bytesAvailable = payload.length; // Ensure the request buffer size will contain the entire request. HTTPServerConfiguration configuration = new HTTPServerConfiguration().withRequestBufferSize(bytesAvailable + 100); - ByteArrayInputStream is = new ByteArrayInputStream(bytes); + ByteArrayInputStream is = new ByteArrayInputStream(payload); PushbackInputStream pushbackInputStream = new PushbackInputStream(is, null); HTTPRequest httpRequest = new HTTPRequest(); @@ -229,37 +247,38 @@ public void parsePreamble() throws Exception { assertEquals(httpRequest.getMethod(), HTTPMethod.GET); assertEquals(httpRequest.getHost(), "localhost"); assertEquals(httpRequest.getPort(), 42); - assertEquals(httpRequest.getContentLength(), 113); - assertEquals(httpRequest.getHeaders().size(), 13); - assertEquals(httpRequest.getHeader("Content-Length"), "113"); + assertEquals(httpRequest.getContentLength(), bodyBytes.length); + assertEquals(httpRequest.getHeaders().size(), contentEncoding.isEmpty() ? 13 : 14); + assertEquals(httpRequest.getHeader("Content-Encoding"), contentEncoding.isEmpty() ? null : contentEncoding); + assertEquals(httpRequest.getHeader("Content-Length"), bodyBytes.length + ""); assertEquals(httpRequest.getHeader("Connection"), "close"); assertEquals(httpRequest.getHeader("Host"), "localhost:42"); for (int i = 1; i <= 10; i++) { assertEquals(httpRequest.getHeader("Header" + i), "Value" + i); } - // Expect 129 bytes left over which is 113 for the body + 16 from the next request - assertEquals(pushbackInputStream.getAvailableBufferedBytesRemaining(), 113 + 16); + // Expect that the body bytes and the next request have been pushed into the pushback buffer + assertEquals(pushbackInputStream.getAvailableBufferedBytesRemaining(), bodyBytes.length + nextRequestBytes.length); // Read the remaining bytes for this request, we should still have some left over. HTTPInputStream httpInputStream = new HTTPInputStream(configuration, httpRequest, pushbackInputStream, -1); byte[] buffer = new byte[1024]; int read = httpInputStream.read(buffer); - assertEquals(read, 113); + assertEquals(read, uncompressedBodyLength); // Another read should return -1 because we are at the end of this request. int nextRead = httpInputStream.read(buffer); assertEquals(nextRead, -1); // The next read from the pushback which will be used by the next request should return the remaining bytes. - assertEquals(pushbackInputStream.getAvailableBufferedBytesRemaining(), 16); + assertEquals(pushbackInputStream.getAvailableBufferedBytesRemaining(), nextRequestBytes.length); int nextRequestRead = pushbackInputStream.read(buffer); - assertEquals(nextRequestRead, 16); + // Expect that we are able to read the bytes for the next request header + assertEquals(nextRequestRead, nextRequestBytes.length); } private void assertEncodedData(String actualValue, String expectedValue, Charset charset) { assertEncodedData(actualValue, expectedValue, charset, charset); - } private void assertEncodedData(String actualValue, String expectedValue, Charset encodingCharset, Charset decodingCharset) {