From 4a8d8b313cd0ff8f0065b76a5b68eaf7c530f1aa Mon Sep 17 00:00:00 2001 From: Daniel DeGroff Date: Tue, 28 Oct 2025 13:36:52 -0600 Subject: [PATCH 01/21] working --- java-http.ipr | 27 +------------------ .../io/fusionauth/http/CompressionTest.java | 27 ++++++++++++++++++- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/java-http.ipr b/java-http.ipr index c470db74..d574a656 100644 --- a/java-http.ipr +++ b/java-http.ipr @@ -1409,31 +1409,6 @@ - - - - - - - @@ -1450,4 +1425,4 @@ - + \ No newline at end of file diff --git a/src/test/java/io/fusionauth/http/CompressionTest.java b/src/test/java/io/fusionauth/http/CompressionTest.java index 045daf9e..4b6339da 100644 --- a/src/test/java/io/fusionauth/http/CompressionTest.java +++ b/src/test/java/io/fusionauth/http/CompressionTest.java @@ -15,12 +15,15 @@ */ package io.fusionauth.http; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.math.BigDecimal; import java.net.URI; import java.net.http.HttpRequest; +import java.net.http.HttpRequest.BodyPublishers; import java.net.http.HttpResponse.BodySubscribers; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -28,7 +31,9 @@ import java.nio.file.Paths; import java.text.DecimalFormat; import java.util.List; +import java.util.zip.DeflaterOutputStream; import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; import java.util.zip.InflaterInputStream; import io.fusionauth.http.HTTPValues.ContentEncodings; @@ -37,6 +42,7 @@ import io.fusionauth.http.server.HTTPHandler; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; +import org.testng.internal.protocols.Input; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNull; import static org.testng.AssertJUnit.fail; @@ -62,6 +68,11 @@ public Object[][] chunkedSchemes() { @Test(dataProvider = "compressedChunkedSchemes") public void compress(String encoding, boolean chunked, String scheme) throws Exception { HTTPHandler handler = (req, res) -> { + + // Ensure we can read the body regardless of the Content-Encoding + String body = new String(req.getInputStream().readAllBytes()); + assertEquals(body, "Hello world!"); + // Testing an indecisive user, can't make up their mind... this is allowed as long as you have not written ay bytes. res.setCompress(true); res.setCompress(false); @@ -94,11 +105,25 @@ public void compress(String encoding, boolean chunked, String scheme) throws Exc } }; + ByteArrayOutputStream baseOutputStream = new ByteArrayOutputStream(); + DeflaterOutputStream out = encoding.equals(ContentEncodings.Deflate) + ? new DeflaterOutputStream(baseOutputStream) + : new GZIPOutputStream(baseOutputStream); + out.write("Hello world!".getBytes(StandardCharsets.UTF_8)); + out.finish(); + byte[] payload = baseOutputStream.toByteArray(); + CountingInstrumenter instrumenter = new CountingInstrumenter(); try (var client = makeClient(scheme, null); var ignore = makeServer(scheme, handler, instrumenter).start()) { URI uri = makeURI(scheme, ""); var response = client.send( - HttpRequest.newBuilder().header(Headers.AcceptEncoding, encoding).uri(uri).GET().build(), + HttpRequest.newBuilder() + // Request the response be compressed using the provided encodings + .header(Headers.AcceptEncoding, encoding) + // Send the request using the same encoding + .header(Headers.ContentEncoding, encoding) + .POST(BodyPublishers.ofInputStream(() -> new ByteArrayInputStream(payload))) + .uri(uri).GET().build(), r -> BodySubscribers.ofInputStream() ); From 1135e86bf7872e8b764ad37d5f2f2b3d55e5d863 Mon Sep 17 00:00:00 2001 From: Daniel DeGroff Date: Tue, 28 Oct 2025 17:51:51 -0600 Subject: [PATCH 02/21] Working --- .../java/io/fusionauth/http/HTTPValues.java | 2 ++ .../fusionauth/http/server/HTTPRequest.java | 34 +++++++++++++------ .../http/server/internal/HTTPWorker.java | 5 +++ .../http/server/io/HTTPInputStream.java | 23 +++++++++++-- .../java/io/fusionauth/http/ChunkedTest.java | 2 +- .../io/fusionauth/http/CompressionTest.java | 9 +++-- 6 files changed, 58 insertions(+), 17 deletions(-) diff --git a/src/main/java/io/fusionauth/http/HTTPValues.java b/src/main/java/io/fusionauth/http/HTTPValues.java index 42697ab4..f5d903f9 100644 --- a/src/main/java/io/fusionauth/http/HTTPValues.java +++ b/src/main/java/io/fusionauth/http/HTTPValues.java @@ -216,6 +216,8 @@ public static final class Headers { public static final String ContentEncoding = "Content-Encoding"; + public static final String ContentEncodingLower = "content-encoding"; + public static final String ContentLength = "Content-Length"; public static final String ContentLengthLower = "content-length"; diff --git a/src/main/java/io/fusionauth/http/server/HTTPRequest.java b/src/main/java/io/fusionauth/http/server/HTTPRequest.java index 341c8d92..19aecedf 100644 --- a/src/main/java/io/fusionauth/http/server/HTTPRequest.java +++ b/src/main/java/io/fusionauth/http/server/HTTPRequest.java @@ -68,6 +68,8 @@ public class HTTPRequest implements Buildable { private final Map attributes = new HashMap<>(); + private final List contentEncodings = new LinkedList<>(); + private final Map cookies = new HashMap<>(); private final List files = new LinkedList<>(); @@ -296,6 +298,15 @@ public void setCharacterEncoding(Charset encoding) { this.encoding = encoding; } + public List getContentEncodings() { + return contentEncodings; + } + + public void setContentEncodings(List encodings) { + this.contentEncodings.clear(); + this.contentEncodings.addAll(encodings); + } + public Long getContentLength() { return contentLength; } @@ -715,7 +726,7 @@ public void setURLParameters(String name, Collection values) { private void decodeHeader(String name, String value) { switch (name) { - case Headers.AcceptEncodingLower: + case Headers.AcceptEncodingLower, Headers.ContentEncodingLower: SortedSet weightedStrings = new TreeSet<>(); String[] parts = value.split(","); int index = 0; @@ -738,11 +749,15 @@ private void decodeHeader(String name, String value) { } // Transfer the Strings in weighted-position order - setAcceptEncodings( - weightedStrings.stream() - .map(WeightedString::value) - .toList() - ); + var result = weightedStrings.stream() + .map(WeightedString::value) + .toList(); + + if (name.equals(Headers.AcceptEncodingLower)) { + setAcceptEncodings(result); + } else { + setContentEncodings(result); + } break; case Headers.AcceptLanguageLower: try { @@ -801,15 +816,12 @@ private void decodeHeader(String name, String value) { } } } else { + this.host = value; if ("http".equalsIgnoreCase(scheme)) { this.port = 80; - } - else if ("https".equalsIgnoreCase(scheme)) { + } else if ("https".equalsIgnoreCase(scheme)) { this.port = 443; - } else { - // fallback, intentionally do nothing } - this.host = value; } break; } diff --git a/src/main/java/io/fusionauth/http/server/internal/HTTPWorker.java b/src/main/java/io/fusionauth/http/server/internal/HTTPWorker.java index 1b150e85..dc40131a 100644 --- a/src/main/java/io/fusionauth/http/server/internal/HTTPWorker.java +++ b/src/main/java/io/fusionauth/http/server/internal/HTTPWorker.java @@ -144,6 +144,11 @@ public void run() { instrumenter.acceptedRequest(); } + // TODO : Content-Encoding, we are currently keeping the PushbackInputStream for the entire keep-alive session. + // Ideally we'd set the InputStream for gzip, deflate, etc lower. + // Also... with Pushback bytes... the bytes may be compressed for the next request. So we'll need to be able to + // read handle this. So perhaps this should all be handled at a high level? + httpInputStream = new HTTPInputStream(configuration, request, inputStream); request.setInputStream(httpInputStream); diff --git a/src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java b/src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java index 448e28d6..9cd60582 100644 --- a/src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java +++ b/src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java @@ -17,7 +17,10 @@ import java.io.IOException; import java.io.InputStream; +import java.util.zip.DeflaterInputStream; +import java.util.zip.GZIPInputStream; +import io.fusionauth.http.HTTPValues.ContentEncodings; import io.fusionauth.http.io.ChunkedInputStream; import io.fusionauth.http.io.PushbackInputStream; import io.fusionauth.http.log.Logger; @@ -161,13 +164,27 @@ public int read(byte[] b, int off, int len) throws IOException { return read; } - private void commit() { + private void commit() throws IOException { committed = true; - // Note that isChunked() should take precedence over the fact that we have a Content-Length. - // - The client should not send both, but in the case they are both present we ignore Content-Length Long contentLength = request.getContentLength(); boolean hasBody = (contentLength != null && contentLength > 0) || request.isChunked(); + +// if (hasBody) { +// // The request may contain more than one value, apply in reverse order. +// for (String contentEncoding : request.getContentEncodings().reversed()) { +// if (contentEncoding.equalsIgnoreCase(ContentEncodings.Deflate)) { +// // Use the default buffer size of 512 +// delegate = new DeflaterInputStream(delegate); +// } else if (contentEncoding.equalsIgnoreCase(ContentEncodings.Gzip)) { +// // Use the default buffer size of 512 +// delegate = new GZIPInputStream(delegate); +// } +// } +// } + + // Note that isChunked() should take precedence over the fact that we have a Content-Length. + // - The client should not send both, but in the case they are both present we ignore Content-Length if (!hasBody) { delegate = InputStream.nullInputStream(); } else if (request.isChunked()) { diff --git a/src/test/java/io/fusionauth/http/ChunkedTest.java b/src/test/java/io/fusionauth/http/ChunkedTest.java index 4a9d2973..0639b4ef 100644 --- a/src/test/java/io/fusionauth/http/ChunkedTest.java +++ b/src/test/java/io/fusionauth/http/ChunkedTest.java @@ -144,7 +144,7 @@ public void chunkedRequest_doNotReadTheInputStream(String scheme) throws Excepti var response = client.send(HttpRequest.newBuilder() .uri(uri) .header(Headers.ContentType, "text/plain") - // Note that using a InputStream baed publisher will caues the JDK to + // Note that using a InputStream based publisher will caues the JDK to // enable Transfer-Encoding: chunked .POST(BodyPublishers.ofInputStream(() -> new ByteArrayInputStream(responseBodyBytes))) diff --git a/src/test/java/io/fusionauth/http/CompressionTest.java b/src/test/java/io/fusionauth/http/CompressionTest.java index 4b6339da..71aec940 100644 --- a/src/test/java/io/fusionauth/http/CompressionTest.java +++ b/src/test/java/io/fusionauth/http/CompressionTest.java @@ -105,10 +105,11 @@ public void compress(String encoding, boolean chunked, String scheme) throws Exc } }; + // Compress the request using the encoding parameter ByteArrayOutputStream baseOutputStream = new ByteArrayOutputStream(); DeflaterOutputStream out = encoding.equals(ContentEncodings.Deflate) - ? new DeflaterOutputStream(baseOutputStream) - : new GZIPOutputStream(baseOutputStream); + ? new DeflaterOutputStream(baseOutputStream, true) + : new GZIPOutputStream(baseOutputStream, true); out.write("Hello world!".getBytes(StandardCharsets.UTF_8)); out.finish(); byte[] payload = baseOutputStream.toByteArray(); @@ -122,6 +123,10 @@ public void compress(String encoding, boolean chunked, String scheme) throws Exc .header(Headers.AcceptEncoding, encoding) // Send the request using the same encoding .header(Headers.ContentEncoding, encoding) + .header(Headers.ContentType, "text/plain") + // Manually set the header since the body is small, the client not turn on chunked. + .header(Headers.TransferEncoding, "chunked") + // A ByteArrayInputStream should cause the request to be chunk encoded .POST(BodyPublishers.ofInputStream(() -> new ByteArrayInputStream(payload))) .uri(uri).GET().build(), r -> BodySubscribers.ofInputStream() From c1c890f72717d532ee332b67f39b960c56603129 Mon Sep 17 00:00:00 2001 From: Daniel DeGroff Date: Wed, 29 Oct 2025 11:54:09 -0600 Subject: [PATCH 03/21] Working --- .../java/io/fusionauth/http/HTTPValues.java | 2 + .../http/io/PushbackInputStream.java | 1 - .../fusionauth/http/server/HTTPRequest.java | 57 ++++++--- .../http/server/internal/HTTPWorker.java | 24 ++++ .../http/server/io/HTTPInputStream.java | 42 ++++--- .../http/server/io/HTTPOutputStream.java | 7 ++ .../io/fusionauth/http/util/HTTPTools.java | 4 +- .../java/io/fusionauth/http/BaseTest.java | 40 +++++++ .../io/fusionauth/http/CompressionTest.java | 109 +++++++++++++----- 9 files changed, 230 insertions(+), 56 deletions(-) diff --git a/src/main/java/io/fusionauth/http/HTTPValues.java b/src/main/java/io/fusionauth/http/HTTPValues.java index f5d903f9..7f21312e 100644 --- a/src/main/java/io/fusionauth/http/HTTPValues.java +++ b/src/main/java/io/fusionauth/http/HTTPValues.java @@ -51,6 +51,8 @@ public static final class ContentEncodings { public static final String Gzip = "gzip"; + public static final String XGzip = "x-gzip"; + private ContentEncodings() { } } diff --git a/src/main/java/io/fusionauth/http/io/PushbackInputStream.java b/src/main/java/io/fusionauth/http/io/PushbackInputStream.java index 206c7779..d8421b6e 100644 --- a/src/main/java/io/fusionauth/http/io/PushbackInputStream.java +++ b/src/main/java/io/fusionauth/http/io/PushbackInputStream.java @@ -38,7 +38,6 @@ public class PushbackInputStream extends InputStream { private int bufferPosition; - public PushbackInputStream(InputStream delegate, Instrumenter instrumenter) { this.delegate = delegate; this.instrumenter = instrumenter; diff --git a/src/main/java/io/fusionauth/http/server/HTTPRequest.java b/src/main/java/io/fusionauth/http/server/HTTPRequest.java index 19aecedf..26890033 100644 --- a/src/main/java/io/fusionauth/http/server/HTTPRequest.java +++ b/src/main/java/io/fusionauth/http/server/HTTPRequest.java @@ -44,6 +44,7 @@ import io.fusionauth.http.FileInfo; import io.fusionauth.http.HTTPMethod; import io.fusionauth.http.HTTPValues.Connections; +import io.fusionauth.http.HTTPValues.ContentEncodings; import io.fusionauth.http.HTTPValues.ContentTypes; import io.fusionauth.http.HTTPValues.Headers; import io.fusionauth.http.HTTPValues.Protocols; @@ -231,7 +232,10 @@ public List getAcceptEncodings() { public void setAcceptEncodings(List encodings) { this.acceptEncodings.clear(); - this.acceptEncodings.addAll(encodings); + // TODO : ? Maybe not worth being defensive here since we likely have many methods on this object that are not null safe? + if (encodings != null) { + this.acceptEncodings.addAll(encodings); + } } /** @@ -302,9 +306,13 @@ public List getContentEncodings() { return contentEncodings; } + // TODO : Note : Should I add 'addContentEncodings' and 'addContentEncoding' to match 'accept*' ? public void setContentEncodings(List encodings) { this.contentEncodings.clear(); - this.contentEncodings.addAll(encodings); + // TODO : ? Maybe not worth being defensive here since we likely have many methods on this object that are not null safe? + if (encodings != null) { + this.contentEncodings.addAll(encodings); + } } public Long getContentLength() { @@ -726,7 +734,7 @@ public void setURLParameters(String name, Collection values) { private void decodeHeader(String name, String value) { switch (name) { - case Headers.AcceptEncodingLower, Headers.ContentEncodingLower: + case Headers.AcceptEncodingLower: SortedSet weightedStrings = new TreeSet<>(); String[] parts = value.split(","); int index = 0; @@ -743,21 +751,19 @@ private void decodeHeader(String name, String value) { weight = Double.parseDouble(weightText); } - WeightedString ws = new WeightedString(parsed.value(), weight, index); + // Content-Encoding values are not case-sensitive + // TODO : Ok to lc here? We are currently just always using equalsIgnoreCase + WeightedString ws = new WeightedString(parsed.value().toLowerCase(), weight, index); weightedStrings.add(ws); index++; } // Transfer the Strings in weighted-position order - var result = weightedStrings.stream() - .map(WeightedString::value) - .toList(); - - if (name.equals(Headers.AcceptEncodingLower)) { - setAcceptEncodings(result); - } else { - setContentEncodings(result); - } + setAcceptEncodings( + weightedStrings.stream() + .map(WeightedString::value) + .toList() + ); break; case Headers.AcceptLanguageLower: try { @@ -771,6 +777,31 @@ private void decodeHeader(String name, String value) { // Ignore the exception and keep the value null } break; + case Headers.ContentEncodingLower: + // TODO : Note that we don't expect more than one Content-Encoding header. We could take the last one we find, + // or try and combine the values. MDN and other places indicate combining may be preferred even though + // it isn't ideal to send multiple headers like this. I tend to think we should just accept the last one. + String[] encodings = value.split(","); + List contentEncodings = new ArrayList<>(1); + int encodingIndex = 0; + for (String encoding : encodings) { + // TODO : Ok to lc here? We are currently just always using equalsIgnoreCase + encoding = encoding.trim().toLowerCase(); + if (encoding.isEmpty()) { + continue; + } + + // The HTTP/1.1 standard recommends that the servers supporting gzip also recognize x-gzip as an alias, for compatibility purposes. + if (encoding.equals(ContentEncodings.XGzip)) { + encoding = ContentEncodings.Gzip; + } + + contentEncodings.add(encoding); + encodingIndex++; + } + + setContentEncodings(contentEncodings); + break; case Headers.ContentTypeLower: this.encoding = null; this.multipart = false; diff --git a/src/main/java/io/fusionauth/http/server/internal/HTTPWorker.java b/src/main/java/io/fusionauth/http/server/internal/HTTPWorker.java index dc40131a..abf72254 100644 --- a/src/main/java/io/fusionauth/http/server/internal/HTTPWorker.java +++ b/src/main/java/io/fusionauth/http/server/internal/HTTPWorker.java @@ -25,6 +25,7 @@ import io.fusionauth.http.HTTPProcessingException; import io.fusionauth.http.HTTPValues; import io.fusionauth.http.HTTPValues.Connections; +import io.fusionauth.http.HTTPValues.ContentEncodings; import io.fusionauth.http.HTTPValues.Headers; import io.fusionauth.http.HTTPValues.Protocols; import io.fusionauth.http.ParseException; @@ -149,6 +150,8 @@ public void run() { // Also... with Pushback bytes... the bytes may be compressed for the next request. So we'll need to be able to // read handle this. So perhaps this should all be handled at a high level? + // HTTPInputStream > Pushback > Decompression > Throughput > Socket + httpInputStream = new HTTPInputStream(configuration, request, inputStream); request.setInputStream(httpInputStream); @@ -449,6 +452,25 @@ private Integer validatePreamble(HTTPRequest request) { request.removeHeader(Headers.ContentLength); } + // TODO : Note that some indicate you can return a 415 based upon the Accept-Encoding header as well if you do not support the request. + // But I don't know that I agree- to me that is just telling the server here are encoding values I support, and you can tell me what you did. + // The spec even says you can ignore the Accept-Encoding header if you want based upon CPU load, or other reasons. + // Not planning to add any validation for Accept-Encoding. + + // Content-Encoding + var contentEncodings = request.getContentEncodings(); + // TODO : If provided, I think we can safely remove the Content-Length header if present? + // Although, I suppose as long as we decompress early, we should still be able to use the Content-Length header as long as + // it represents the un-compressed payload. + for (var encoding : contentEncodings) { + // We only support gzip and deflate. + // TODO : Ensure we use the same equals check here as other places, I think we should lowercase during parse, and then expect these to be lc. + if (!encoding.equals(ContentEncodings.Gzip) && !encoding.equals(ContentEncodings.Deflate)) { + // TODO : Add log statement + return Status.UnsupportedMediaType; + } + } + return null; } @@ -467,6 +489,8 @@ private enum CloseSocketReason { private static class Status { public static final int BadRequest = 400; + public static final int UnsupportedMediaType = 415; + public static final int HTTPVersionNotSupported = 505; public static final int InternalServerError = 500; diff --git a/src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java b/src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java index 9cd60582..f0045509 100644 --- a/src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java +++ b/src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java @@ -17,8 +17,8 @@ import java.io.IOException; import java.io.InputStream; -import java.util.zip.DeflaterInputStream; import java.util.zip.GZIPInputStream; +import java.util.zip.InflaterInputStream; import io.fusionauth.http.HTTPValues.ContentEncodings; import io.fusionauth.http.io.ChunkedInputStream; @@ -170,18 +170,20 @@ private void commit() throws IOException { Long contentLength = request.getContentLength(); boolean hasBody = (contentLength != null && contentLength > 0) || request.isChunked(); -// if (hasBody) { -// // The request may contain more than one value, apply in reverse order. -// for (String contentEncoding : request.getContentEncodings().reversed()) { -// if (contentEncoding.equalsIgnoreCase(ContentEncodings.Deflate)) { -// // Use the default buffer size of 512 -// delegate = new DeflaterInputStream(delegate); -// } else if (contentEncoding.equalsIgnoreCase(ContentEncodings.Gzip)) { -// // Use the default buffer size of 512 -// delegate = new GZIPInputStream(delegate); -// } -// } -// } + // Request order of operations: + // - "hello world" -> compress -> chunked. + + // Current: + // Chunked: + // HTTPInputStream > Chunked > Pushback > Throughput > Socket + // Fixed: + // HTTPInputStream > Pushback > Throughput > Socket + + // New: + // Chunked + // HTTPInputStream > Chunked > Pushback > Decompress > Throughput > Socket + // Fixed + // HTTPInputStream > Pushback > Decompress > Throughput > Socket // Note that isChunked() should take precedence over the fact that we have a Content-Length. // - The client should not send both, but in the case they are both present we ignore Content-Length @@ -198,5 +200,19 @@ private void commit() throws IOException { } else { logger.trace("Client indicated it was NOT sending an entity-body in the request"); } + + // TODO : Note I could leave this alone, but when we parse the header we can lower case these values and then remove the equalsIgnoreCase here? + // Seems like ideally we would normalize them to lowercase earlier. + if (hasBody) { + // The request may contain more than one value, apply in reverse order. + // - These are both using the default 512 buffer size. + for (String contentEncoding : request.getContentEncodings().reversed()) { + if (contentEncoding.equalsIgnoreCase(ContentEncodings.Deflate)) { + delegate = new InflaterInputStream(delegate); + } else if (contentEncoding.equalsIgnoreCase(ContentEncodings.Gzip)) { + delegate = new GZIPInputStream(delegate); + } + } + } } } diff --git a/src/main/java/io/fusionauth/http/server/io/HTTPOutputStream.java b/src/main/java/io/fusionauth/http/server/io/HTTPOutputStream.java index 7a29a327..0f089e90 100644 --- a/src/main/java/io/fusionauth/http/server/io/HTTPOutputStream.java +++ b/src/main/java/io/fusionauth/http/server/io/HTTPOutputStream.java @@ -128,6 +128,9 @@ public void reset() { */ public boolean willCompress() { if (compress) { + // TODO : Note I could leave this alone, but when we parse the header we can lower case these values and then remove the equalsIgnoreCase here? + // Seems like ideally we would normalize them to lowercase earlier. + // Hmm.. sems like we have to in theory since someone could call setAcceptEncodings later, or addAcceptEncodings? for (String encoding : acceptEncodings) { if (encoding.equalsIgnoreCase(ContentEncodings.Gzip)) { return true; @@ -191,7 +194,11 @@ private void commit(boolean closing) throws IOException { response.setContentLength(0L); } else { // 204 status is specifically "No Content" so we shouldn't write the content-encoding and vary headers if the status is 204 + // TODO : Compress by default is on by default. But it looks like we don't actually compress unless you also send in an Accept-Encoding header? if (compress && !twoOhFour) { + // TODO : Note I could leave this alone, but when we parse the header we can lower case these values and then remove the equalsIgnoreCase here? + // Seems like ideally we would normalize them to lowercase earlier. + // Hmm.. sems like we have to in theory since someone could call setAcceptEncodings later, or addAcceptEncodings? for (String encoding : acceptEncodings) { if (encoding.equalsIgnoreCase(ContentEncodings.Gzip)) { response.setHeader(Headers.ContentEncoding, ContentEncodings.Gzip); diff --git a/src/main/java/io/fusionauth/http/util/HTTPTools.java b/src/main/java/io/fusionauth/http/util/HTTPTools.java index c38356f0..0fe36017 100644 --- a/src/main/java/io/fusionauth/http/util/HTTPTools.java +++ b/src/main/java/io/fusionauth/http/util/HTTPTools.java @@ -92,8 +92,8 @@ public static boolean isHexadecimalCharacter(byte ch) { */ public static boolean isTokenCharacter(byte ch) { return ch == '!' || ch == '#' || ch == '$' || ch == '%' || ch == '&' || ch == '\'' || ch == '*' || ch == '+' || ch == '-' || ch == '.' || - ch == '^' || ch == '_' || ch == '`' || ch == '|' || ch == '~' || (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') || - (ch >= '0' && ch <= '9'); + ch == '^' || ch == '_' || ch == '`' || ch == '|' || ch == '~' || (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') || + (ch >= '0' && ch <= '9'); } /** diff --git a/src/test/java/io/fusionauth/http/BaseTest.java b/src/test/java/io/fusionauth/http/BaseTest.java index 447bbed8..a2b73792 100644 --- a/src/test/java/io/fusionauth/http/BaseTest.java +++ b/src/test/java/io/fusionauth/http/BaseTest.java @@ -15,6 +15,8 @@ */ package io.fusionauth.http; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -51,6 +53,10 @@ import java.util.Map; import java.util.UUID; import java.util.stream.Collectors; +import java.util.zip.DeflaterOutputStream; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; +import java.util.zip.InflaterInputStream; import io.fusionauth.http.HTTPValues.Connections; import io.fusionauth.http.log.FileLogger; @@ -407,6 +413,33 @@ protected void assertHTTPResponseEquals(Socket socket, String expectedResponse) } } + protected byte[] deflate(byte[] bytes) throws Exception { + ByteArrayOutputStream baseOutputStream = new ByteArrayOutputStream(); + try (DeflaterOutputStream out = new DeflaterOutputStream(baseOutputStream, true)) { + out.write(bytes); + out.flush(); + out.finish(); + return baseOutputStream.toByteArray(); + } + } + + protected byte[] gzip(byte[] bytes) throws Exception { + ByteArrayOutputStream baseOutputStream = new ByteArrayOutputStream(); + try (DeflaterOutputStream out = new GZIPOutputStream(baseOutputStream, true)) { + out.write(bytes); + out.flush(); + out.finish(); + return baseOutputStream.toByteArray(); + } + } + + protected byte[] inflate(byte[] bytes) throws Exception { + ByteArrayInputStream baseInputStream = new ByteArrayInputStream(bytes); + try (InflaterInputStream in = new InflaterInputStream(baseInputStream)) { + return in.readAllBytes(); + } + } + protected void printf(String format, Object... args) { if (verbose) { System.out.printf(SystemOutPrefix + format, args); @@ -426,6 +459,13 @@ protected void sleep(long millis) { } } + protected byte[] ungzip(byte[] bytes) throws Exception { + ByteArrayInputStream baseInputStream = new ByteArrayInputStream(bytes); + try (InflaterInputStream in = new GZIPInputStream(baseInputStream)) { + return in.readAllBytes(); + } + } + /** * Verifies that the chain certificates can be validated up to the supplied root certificate. See * {@link CertPathValidator#validate(CertPath, CertPathParameters)} for details. diff --git a/src/test/java/io/fusionauth/http/CompressionTest.java b/src/test/java/io/fusionauth/http/CompressionTest.java index 71aec940..a1002859 100644 --- a/src/test/java/io/fusionauth/http/CompressionTest.java +++ b/src/test/java/io/fusionauth/http/CompressionTest.java @@ -16,7 +16,6 @@ package io.fusionauth.http; import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -31,9 +30,7 @@ import java.nio.file.Paths; import java.text.DecimalFormat; import java.util.List; -import java.util.zip.DeflaterOutputStream; import java.util.zip.GZIPInputStream; -import java.util.zip.GZIPOutputStream; import java.util.zip.InflaterInputStream; import io.fusionauth.http.HTTPValues.ContentEncodings; @@ -42,8 +39,8 @@ import io.fusionauth.http.server.HTTPHandler; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -import org.testng.internal.protocols.Input; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; import static org.testng.AssertJUnit.fail; @@ -66,7 +63,7 @@ public Object[][] chunkedSchemes() { } @Test(dataProvider = "compressedChunkedSchemes") - public void compress(String encoding, boolean chunked, String scheme) throws Exception { + public void compress(String contentEncoding, String acceptEncoding, boolean chunked, String scheme) throws Exception { HTTPHandler handler = (req, res) -> { // Ensure we can read the body regardless of the Content-Encoding @@ -106,13 +103,30 @@ public void compress(String encoding, boolean chunked, String scheme) throws Exc }; // Compress the request using the encoding parameter - ByteArrayOutputStream baseOutputStream = new ByteArrayOutputStream(); - DeflaterOutputStream out = encoding.equals(ContentEncodings.Deflate) - ? new DeflaterOutputStream(baseOutputStream, true) - : new GZIPOutputStream(baseOutputStream, true); - out.write("Hello world!".getBytes(StandardCharsets.UTF_8)); - out.finish(); - byte[] payload = baseOutputStream.toByteArray(); + byte[] body = "Hello world!".getBytes(StandardCharsets.UTF_8); + byte[] compressedBody = body; + + var requestEncodings = contentEncoding.toLowerCase().trim().split(","); + for (String part : requestEncodings) { + String encoding = part.trim(); + compressedBody = encoding.equals(ContentEncodings.Deflate) + ? deflate(compressedBody) + : gzip(compressedBody); + } + + var payload = compressedBody; + byte[] uncompressedBody = compressedBody; + + // Sanity check on round trip compress/decompress + for (int i = requestEncodings.length - 1; i >= 0; i--) { + String encoding = requestEncodings[i].trim(); + uncompressedBody = encoding.equals(ContentEncodings.Deflate) + ? inflate(uncompressedBody) + : ungzip(uncompressedBody); + } + + assertEquals(uncompressedBody, body); + assertEquals(new String(body), "Hello world!"); CountingInstrumenter instrumenter = new CountingInstrumenter(); try (var client = makeClient(scheme, null); var ignore = makeServer(scheme, handler, instrumenter).start()) { @@ -120,25 +134,34 @@ public void compress(String encoding, boolean chunked, String scheme) throws Exc var response = client.send( HttpRequest.newBuilder() // Request the response be compressed using the provided encodings - .header(Headers.AcceptEncoding, encoding) + .header(Headers.AcceptEncoding, acceptEncoding) // Send the request using the same encoding - .header(Headers.ContentEncoding, encoding) + .header(Headers.ContentEncoding, contentEncoding) .header(Headers.ContentType, "text/plain") // Manually set the header since the body is small, the client not turn on chunked. .header(Headers.TransferEncoding, "chunked") + .uri(uri) // A ByteArrayInputStream should cause the request to be chunk encoded .POST(BodyPublishers.ofInputStream(() -> new ByteArrayInputStream(payload))) - .uri(uri).GET().build(), + .build(), r -> BodySubscribers.ofInputStream() ); + assertEquals(response.statusCode(), 200); + String expectedResponseEncoding = null; + for (String part : acceptEncoding.toLowerCase().trim().split(",")) { + expectedResponseEncoding = part.trim(); + break; + } + + assertNotNull(expectedResponseEncoding); + assertEquals(response.headers().firstValue(Headers.ContentEncoding).orElse(null), expectedResponseEncoding); + var result = new String( - encoding.equals(ContentEncodings.Deflate) + expectedResponseEncoding.equals(ContentEncodings.Deflate) ? new InflaterInputStream(response.body()).readAllBytes() : new GZIPInputStream(response.body()).readAllBytes(), StandardCharsets.UTF_8); - assertEquals(response.headers().firstValue(Headers.ContentEncoding).orElse(null), encoding); - assertEquals(response.statusCode(), 200); assertEquals(result, Files.readString(file)); } } @@ -270,23 +293,55 @@ public void compress_onByDefault(String encoding, boolean chunked, String scheme @DataProvider(name = "compressedChunkedSchemes") public Object[][] compressedChunkedSchemes() { return new Object[][]{ - // encoding, chunked, schema + // Content-Encoding, Accept-Encoding, chunked, schema // Chunked http - {ContentEncodings.Deflate, true, "http"}, - {ContentEncodings.Gzip, true, "http"}, + {"deflate", "deflate", true, "http"}, + {"gzip", "gzip", true, "http"}, // Chunked https - {ContentEncodings.Deflate, true, "https"}, - {ContentEncodings.Gzip, true, "https"}, + {"deflate", "deflate", true, "https"}, + {"gzip", "gzip", true, "https"}, // Non chunked http - {ContentEncodings.Deflate, false, "http"}, - {ContentEncodings.Gzip, false, "http"}, + {"deflate", "deflate", false, "http"}, + {"gzip", "gzip", false, "http"}, // Non chunked https - {ContentEncodings.Deflate, false, "https"}, - {ContentEncodings.Gzip, false, "https"} + {"deflate", "deflate", false, "https"}, + {"gzip", "gzip", false, "https"}, + + // UC, and mixed case, http + {"Deflate", "Deflate", true, "http"}, + {"Gzip", "Gzip", true, "http"}, + + // UC, and mixed case, https + {"Deflate", "Deflate", true, "https"}, + {"Gzip", "Gzip", true, "https"}, + + // Multiple accept values, expect to use the first, http + {"deflate", "deflate, gzip", true, "http"}, + {"gzip", "gzip, deflate", true, "http"}, + + // Multiple accept values, expect to use the first, https + {"deflate", "deflate, gzip", true, "https"}, + {"gzip", "gzip, deflate", true, "https"}, + + // Multiple request values, this means we will use multiple passes of compression, http + {"deflate", "deflate, gzip", true, "https"}, + {"gzip", "gzip, deflate", true, "https"}, + + // Multiple request values, this means we will use multiple passes of compression, https + {"deflate, gzip", "deflate, gzip", true, "https"}, + {"gzip, deflate", "gzip, deflate", true, "https"}, + + // x-gzip alias, http + {"deflate, gzip", "deflate, gzip", true, "https"}, + {"gzip, deflate", "gzip, deflate", true, "https"}, + + // x-gzip alias, https + {"x-gzip", "deflate, gzip", true, "https"}, + {"x-gzip", "gzip, deflate", true, "https"}, }; } From 00943bf1cd41f22ca0e16700749faa162683eeff Mon Sep 17 00:00:00 2001 From: Daniel DeGroff Date: Wed, 29 Oct 2025 15:13:58 -0600 Subject: [PATCH 04/21] Working --- README.md | 2 + .../fusionauth/http/server/HTTPRequest.java | 29 +- .../http/server/internal/HTTPWorker.java | 27 +- .../http/server/io/HTTPInputStream.java | 21 +- .../http/server/io/HTTPOutputStream.java | 8 +- .../io/fusionauth/http/CompressionTest.java | 289 ++++++++++-------- 6 files changed, 208 insertions(+), 168 deletions(-) diff --git a/README.md b/README.md index 94e0bda2..34b0a28f 100644 --- a/README.md +++ b/README.md @@ -231,6 +231,8 @@ The general requirements and roadmap are as follows: ### Server tasks * [x] Basic HTTP 1.1 +* [x] Support Accept-Encoding (gzip, deflate) +* [x] Support Content-Encoding (gzip, deflate) * [x] Support Keep-Alive * [x] Support Expect-Continue 100 * [x] Support chunked request diff --git a/src/main/java/io/fusionauth/http/server/HTTPRequest.java b/src/main/java/io/fusionauth/http/server/HTTPRequest.java index 26890033..48912dbf 100644 --- a/src/main/java/io/fusionauth/http/server/HTTPRequest.java +++ b/src/main/java/io/fusionauth/http/server/HTTPRequest.java @@ -150,6 +150,14 @@ public void addAcceptEncodings(List encodings) { this.acceptEncodings.addAll(encodings); } + public void addContentEncoding(String encoding) { + this.contentEncodings.add(encoding); + } + + public void addContentEncodings(List encodings) { + this.contentEncodings.addAll(encodings); + } + public void addCookies(Cookie... cookies) { for (Cookie cookie : cookies) { this.cookies.put(cookie.name, cookie); @@ -232,10 +240,7 @@ public List getAcceptEncodings() { public void setAcceptEncodings(List encodings) { this.acceptEncodings.clear(); - // TODO : ? Maybe not worth being defensive here since we likely have many methods on this object that are not null safe? - if (encodings != null) { - this.acceptEncodings.addAll(encodings); - } + this.acceptEncodings.addAll(encodings); } /** @@ -306,13 +311,9 @@ public List getContentEncodings() { return contentEncodings; } - // TODO : Note : Should I add 'addContentEncodings' and 'addContentEncoding' to match 'accept*' ? public void setContentEncodings(List encodings) { this.contentEncodings.clear(); - // TODO : ? Maybe not worth being defensive here since we likely have many methods on this object that are not null safe? - if (encodings != null) { - this.contentEncodings.addAll(encodings); - } + this.contentEncodings.addAll(encodings); } public Long getContentLength() { @@ -751,9 +752,7 @@ private void decodeHeader(String name, String value) { weight = Double.parseDouble(weightText); } - // Content-Encoding values are not case-sensitive - // TODO : Ok to lc here? We are currently just always using equalsIgnoreCase - WeightedString ws = new WeightedString(parsed.value().toLowerCase(), weight, index); + WeightedString ws = new WeightedString(parsed.value(), weight, index); weightedStrings.add(ws); index++; } @@ -778,15 +777,11 @@ private void decodeHeader(String name, String value) { } break; case Headers.ContentEncodingLower: - // TODO : Note that we don't expect more than one Content-Encoding header. We could take the last one we find, - // or try and combine the values. MDN and other places indicate combining may be preferred even though - // it isn't ideal to send multiple headers like this. I tend to think we should just accept the last one. String[] encodings = value.split(","); List contentEncodings = new ArrayList<>(1); int encodingIndex = 0; for (String encoding : encodings) { - // TODO : Ok to lc here? We are currently just always using equalsIgnoreCase - encoding = encoding.trim().toLowerCase(); + encoding = encoding.trim(); if (encoding.isEmpty()) { continue; } diff --git a/src/main/java/io/fusionauth/http/server/internal/HTTPWorker.java b/src/main/java/io/fusionauth/http/server/internal/HTTPWorker.java index abf72254..9567a2d2 100644 --- a/src/main/java/io/fusionauth/http/server/internal/HTTPWorker.java +++ b/src/main/java/io/fusionauth/http/server/internal/HTTPWorker.java @@ -145,13 +145,6 @@ public void run() { instrumenter.acceptedRequest(); } - // TODO : Content-Encoding, we are currently keeping the PushbackInputStream for the entire keep-alive session. - // Ideally we'd set the InputStream for gzip, deflate, etc lower. - // Also... with Pushback bytes... the bytes may be compressed for the next request. So we'll need to be able to - // read handle this. So perhaps this should all be handled at a high level? - - // HTTPInputStream > Pushback > Decompression > Throughput > Socket - httpInputStream = new HTTPInputStream(configuration, request, inputStream); request.setInputStream(httpInputStream); @@ -452,21 +445,19 @@ private Integer validatePreamble(HTTPRequest request) { request.removeHeader(Headers.ContentLength); } - // TODO : Note that some indicate you can return a 415 based upon the Accept-Encoding header as well if you do not support the request. - // But I don't know that I agree- to me that is just telling the server here are encoding values I support, and you can tell me what you did. - // The spec even says you can ignore the Accept-Encoding header if you want based upon CPU load, or other reasons. - // Not planning to add any validation for Accept-Encoding. - // Content-Encoding var contentEncodings = request.getContentEncodings(); // TODO : If provided, I think we can safely remove the Content-Length header if present? // Although, I suppose as long as we decompress early, we should still be able to use the Content-Length header as long as // it represents the un-compressed payload. + // Maybe write a test to prove that we can compress with a fixed-length w/ a Content-Length header? + // Current support is for gzip and deflate. for (var encoding : contentEncodings) { - // We only support gzip and deflate. - // TODO : Ensure we use the same equals check here as other places, I think we should lowercase during parse, and then expect these to be lc. - if (!encoding.equals(ContentEncodings.Gzip) && !encoding.equals(ContentEncodings.Deflate)) { - // TODO : Add log statement + if (!encoding.equalsIgnoreCase(ContentEncodings.Gzip) && !encoding.equalsIgnoreCase(ContentEncodings.Deflate)) { + // Note that while we do not expect multiple Content-Encoding headers, the last one will be used. For good measure, + // use the last one in the debug message as well. + var contentEncodingHeader = request.getHeaders(Headers.ContentEncoding).getLast(); + logger.debug("Invalid request. The Content-Type header contains an un-supported value. [{}]", contentEncodingHeader); return Status.UnsupportedMediaType; } } @@ -489,10 +480,10 @@ private enum CloseSocketReason { private static class Status { public static final int BadRequest = 400; - public static final int UnsupportedMediaType = 415; - public static final int HTTPVersionNotSupported = 505; public static final int InternalServerError = 500; + + public static final int UnsupportedMediaType = 415; } } diff --git a/src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java b/src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java index f0045509..92b412d6 100644 --- a/src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java +++ b/src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java @@ -54,7 +54,7 @@ public class HTTPInputStream extends InputStream { private boolean closed; - private boolean committed; + private boolean initialized; private InputStream delegate; @@ -141,8 +141,8 @@ public int read(byte[] b, int off, int len) throws IOException { return -1; } - if (!committed) { - commit(); + if (!initialized) { + initialize(); } // When we have a fixed length request, read beyond the remainingBytes if possible. @@ -164,8 +164,8 @@ public int read(byte[] b, int off, int len) throws IOException { return read; } - private void commit() throws IOException { - committed = true; + private void initialize() throws IOException { + initialized = true; Long contentLength = request.getContentLength(); boolean hasBody = (contentLength != null && contentLength > 0) || request.isChunked(); @@ -201,6 +201,17 @@ private void commit() throws IOException { logger.trace("Client indicated it was NOT sending an entity-body in the request"); } + // Those who push back: + // HTTPInputStream when fixed + // ChunkedInputStream when chunked + // Preamble parser + + // HTTPInputStream (this) > Pushback (delegate) > Throughput > Socket + // HTTPInputStream (this) > Chunked (delegate) > Pushback > Throughput > Socket + + // HTTPInputStream (this) > Pushback > Decompress > Throughput > Socket + // HTTPInputStream (this) > Pushback > Decompress > Chunked > Throughput > Socket + // TODO : Note I could leave this alone, but when we parse the header we can lower case these values and then remove the equalsIgnoreCase here? // Seems like ideally we would normalize them to lowercase earlier. if (hasBody) { diff --git a/src/main/java/io/fusionauth/http/server/io/HTTPOutputStream.java b/src/main/java/io/fusionauth/http/server/io/HTTPOutputStream.java index 0f089e90..6c3e51ba 100644 --- a/src/main/java/io/fusionauth/http/server/io/HTTPOutputStream.java +++ b/src/main/java/io/fusionauth/http/server/io/HTTPOutputStream.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024, FusionAuth, All Rights Reserved + * Copyright (c) 2024-2025, FusionAuth, All Rights Reserved * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -128,9 +128,6 @@ public void reset() { */ public boolean willCompress() { if (compress) { - // TODO : Note I could leave this alone, but when we parse the header we can lower case these values and then remove the equalsIgnoreCase here? - // Seems like ideally we would normalize them to lowercase earlier. - // Hmm.. sems like we have to in theory since someone could call setAcceptEncodings later, or addAcceptEncodings? for (String encoding : acceptEncodings) { if (encoding.equalsIgnoreCase(ContentEncodings.Gzip)) { return true; @@ -196,9 +193,6 @@ private void commit(boolean closing) throws IOException { // 204 status is specifically "No Content" so we shouldn't write the content-encoding and vary headers if the status is 204 // TODO : Compress by default is on by default. But it looks like we don't actually compress unless you also send in an Accept-Encoding header? if (compress && !twoOhFour) { - // TODO : Note I could leave this alone, but when we parse the header we can lower case these values and then remove the equalsIgnoreCase here? - // Seems like ideally we would normalize them to lowercase earlier. - // Hmm.. sems like we have to in theory since someone could call setAcceptEncodings later, or addAcceptEncodings? for (String encoding : acceptEncodings) { if (encoding.equalsIgnoreCase(ContentEncodings.Gzip)) { response.setHeader(Headers.ContentEncoding, ContentEncodings.Gzip); diff --git a/src/test/java/io/fusionauth/http/CompressionTest.java b/src/test/java/io/fusionauth/http/CompressionTest.java index a1002859..58ef7ae0 100644 --- a/src/test/java/io/fusionauth/http/CompressionTest.java +++ b/src/test/java/io/fusionauth/http/CompressionTest.java @@ -55,94 +55,78 @@ public class CompressionTest extends BaseTest { @DataProvider(name = "chunkedSchemes") public Object[][] chunkedSchemes() { return new Object[][]{ - {true, "http"}, - {true, "https"}, - {false, "http"}, - {false, "https"} + {"http", true}, + {"http", false}, + {"https", true}, + {"https", false} }; } @Test(dataProvider = "compressedChunkedSchemes") - public void compress(String contentEncoding, String acceptEncoding, boolean chunked, String scheme) throws Exception { + public void compress(String scheme, String contentEncoding, String acceptEncoding) throws Exception { HTTPHandler handler = (req, res) -> { - // Ensure we can read the body regardless of the Content-Encoding String body = new String(req.getInputStream().readAllBytes()); assertEquals(body, "Hello world!"); - // Testing an indecisive user, can't make up their mind... this is allowed as long as you have not written ay bytes. - res.setCompress(true); - res.setCompress(false); - res.setCompress(true); - res.setCompress(false); - res.setCompress(true); - res.setHeader(Headers.ContentType, "text/plain"); res.setStatus(200); // Technically, this is ignored anytime compression is used, but we are testing if folks don't set it here - if (!chunked) { - res.setContentLength(Files.size(file)); - } + res.setContentLength(Files.size(file)); try (InputStream is = Files.newInputStream(file)) { OutputStream outputStream = res.getOutputStream(); is.transferTo(outputStream); - - // Try to call setCompress, expect an exception - we cannot change modes once we've written to the OutputStream. - try { - res.setCompress(false); - fail("Expected setCompress(false) to fail hard!"); - } catch (IllegalStateException expected) { - } - outputStream.close(); } catch (IOException e) { throw new RuntimeException(e); } }; - // Compress the request using the encoding parameter - byte[] body = "Hello world!".getBytes(StandardCharsets.UTF_8); - byte[] compressedBody = body; + String bodyString = "Hello world!"; + byte[] bodyBytes = bodyString.getBytes(StandardCharsets.UTF_8); + var payload = bodyBytes; + + if (!contentEncoding.isEmpty()) { + var requestEncodings = contentEncoding.toLowerCase().trim().split(","); + for (String part : requestEncodings) { + String encoding = part.trim(); + payload = encoding.equals(ContentEncodings.Deflate) + ? deflate(payload) + : gzip(payload); + } - var requestEncodings = contentEncoding.toLowerCase().trim().split(","); - for (String part : requestEncodings) { - String encoding = part.trim(); - compressedBody = encoding.equals(ContentEncodings.Deflate) - ? deflate(compressedBody) - : gzip(compressedBody); - } + byte[] uncompressedBody = payload; - var payload = compressedBody; - byte[] uncompressedBody = compressedBody; + // Sanity check on round trip compress/decompress + for (int i = requestEncodings.length - 1; i >= 0; i--) { + String encoding = requestEncodings[i].trim(); + uncompressedBody = encoding.equals(ContentEncodings.Deflate) + ? inflate(uncompressedBody) + : ungzip(uncompressedBody); + } - // Sanity check on round trip compress/decompress - for (int i = requestEncodings.length - 1; i >= 0; i--) { - String encoding = requestEncodings[i].trim(); - uncompressedBody = encoding.equals(ContentEncodings.Deflate) - ? inflate(uncompressedBody) - : ungzip(uncompressedBody); + assertEquals(uncompressedBody, bodyBytes); + assertEquals(new String(bodyBytes), bodyString); } - assertEquals(uncompressedBody, body); - assertEquals(new String(body), "Hello world!"); + var requestPayload = payload; + // Make the request CountingInstrumenter instrumenter = new CountingInstrumenter(); try (var client = makeClient(scheme, null); var ignore = makeServer(scheme, handler, instrumenter).start()) { URI uri = makeURI(scheme, ""); var response = client.send( HttpRequest.newBuilder() - // Request the response be compressed using the provided encodings + .uri(uri) .header(Headers.AcceptEncoding, acceptEncoding) - // Send the request using the same encoding .header(Headers.ContentEncoding, contentEncoding) .header(Headers.ContentType, "text/plain") - // Manually set the header since the body is small, the client not turn on chunked. + // In general using a BodyPublishers.ofInputStream causes the client to use chunked transfer encoding. + // - Manually set the header because the body is small, and it may not chunk it otherwise. .header(Headers.TransferEncoding, "chunked") - .uri(uri) - // A ByteArrayInputStream should cause the request to be chunk encoded - .POST(BodyPublishers.ofInputStream(() -> new ByteArrayInputStream(payload))) + .POST(BodyPublishers.ofInputStream(() -> new ByteArrayInputStream(requestPayload))) .build(), r -> BodySubscribers.ofInputStream() ); @@ -155,12 +139,19 @@ public void compress(String contentEncoding, String acceptEncoding, boolean chun } assertNotNull(expectedResponseEncoding); - assertEquals(response.headers().firstValue(Headers.ContentEncoding).orElse(null), expectedResponseEncoding); - var result = new String( - expectedResponseEncoding.equals(ContentEncodings.Deflate) - ? new InflaterInputStream(response.body()).readAllBytes() - : new GZIPInputStream(response.body()).readAllBytes(), StandardCharsets.UTF_8); + String result; + InputStream responseInputStream = response.body(); + + if (expectedResponseEncoding.isEmpty()) { + result = new String(responseInputStream.readAllBytes()); + } else { + assertEquals(response.headers().firstValue(Headers.ContentEncoding).orElse(null), expectedResponseEncoding); + result = new String( + expectedResponseEncoding.equals(ContentEncodings.Deflate) + ? new InflaterInputStream(responseInputStream).readAllBytes() + : new GZIPInputStream(responseInputStream).readAllBytes(), StandardCharsets.UTF_8); + } assertEquals(result, Files.readString(file)); } @@ -250,103 +241,112 @@ public void compressPerformance() throws Exception { } } - @Test(dataProvider = "compressedChunkedSchemes") - public void compress_onByDefault(String encoding, boolean chunked, String scheme) throws Exception { + @Test(dataProvider = "schemes") + public void compressWithContentLength(String scheme) throws Exception { + // Use case: Compress the request w/out using chunked transfer encoding. + // - The JDK rest client is hard to predict sometimes, but I think the body is small enough it won't chunk it up. + String bodyString = "Hello world!"; + byte[] bodyBytes = bodyString.getBytes(StandardCharsets.UTF_8); + var payload = gzip(bodyBytes); + HTTPHandler handler = (req, res) -> { - // Use case, do not call response.setCompress(true) + + assertEquals(req.isChunked(), false); + + // We forced a Content-Length by telling the JDK client not to chunk, so it will be present. It will + // be used in theory - and it should be ok as long as we are using it to count compressed bytes? + // TODO : Once I add the push back tests for compression, we'll see if this still works. We may + // need to manually remove the Content-Length header when we know the body to be compressed. + var contentLength = req.getHeader(Headers.ContentLength); + assertEquals(contentLength, payload.length + ""); + + String body = new String(req.getInputStream().readAllBytes()); + assertEquals(body, bodyString); + res.setHeader(Headers.ContentType, "text/plain"); res.setStatus(200); + }; - // Technically, this is ignored anytime compression is used, but we are testing if folks don't set it here - if (!chunked) { - res.setContentLength(Files.size(file)); - } + // This publisher should cause a fixed length request. + var bodyPublisher = BodyPublishers.ofByteArray(payload); - try (InputStream is = Files.newInputStream(file)) { - OutputStream outputStream = res.getOutputStream(); - is.transferTo(outputStream); - outputStream.close(); - } catch (IOException e) { - throw new RuntimeException(e); - } - }; + try (var client = makeClient(scheme, null); + var ignore = makeServer(scheme, handler, null).start()) { - CountingInstrumenter instrumenter = new CountingInstrumenter(); - try (var client = makeClient(scheme, null); var ignore = makeServer(scheme, handler, instrumenter).start()) { URI uri = makeURI(scheme, ""); var response = client.send( - HttpRequest.newBuilder().header(Headers.AcceptEncoding, encoding).uri(uri).GET().build(), - r -> BodySubscribers.ofInputStream() + HttpRequest.newBuilder() + .uri(uri) + .header(Headers.ContentEncoding, "gzip") + .header(Headers.ContentType, "text/plain") + .POST(bodyPublisher) + .build(), + r -> BodySubscribers.ofString(StandardCharsets.UTF_8) ); - var result = new String( - encoding.equals(ContentEncodings.Deflate) - ? new InflaterInputStream(response.body()).readAllBytes() - : new GZIPInputStream(response.body()).readAllBytes(), StandardCharsets.UTF_8); - - assertEquals(response.headers().firstValue(Headers.ContentEncoding).orElse(null), encoding); assertEquals(response.statusCode(), 200); - assertEquals(result, Files.readString(file)); + assertEquals(response.body(), ""); } } @DataProvider(name = "compressedChunkedSchemes") public Object[][] compressedChunkedSchemes() { - return new Object[][]{ - // Content-Encoding, Accept-Encoding, chunked, schema + Object[][] compressOptions = new Object[][]{ + // Content-Encoding, Accept-Encoding - // Chunked http - {"deflate", "deflate", true, "http"}, - {"gzip", "gzip", true, "http"}, + // No compression on request, or response + {"", ""}, - // Chunked https - {"deflate", "deflate", true, "https"}, - {"gzip", "gzip", true, "https"}, + // Only request + {"deflate", ""}, + {"gzip", ""}, - // Non chunked http - {"deflate", "deflate", false, "http"}, - {"gzip", "gzip", false, "http"}, + // Only response + {"", "deflate"}, + {"", "gzip"}, - // Non chunked https - {"deflate", "deflate", false, "https"}, - {"gzip", "gzip", false, "https"}, + // Same on request and response + {"deflate", "deflate"}, + {"gzip", "gzip"}, // UC, and mixed case, http - {"Deflate", "Deflate", true, "http"}, - {"Gzip", "Gzip", true, "http"}, - - // UC, and mixed case, https - {"Deflate", "Deflate", true, "https"}, - {"Gzip", "Gzip", true, "https"}, + {"Deflate", "Deflate"}, + {"Gzip", "Gzip"}, // Multiple accept values, expect to use the first, http - {"deflate", "deflate, gzip", true, "http"}, - {"gzip", "gzip, deflate", true, "http"}, - - // Multiple accept values, expect to use the first, https - {"deflate", "deflate, gzip", true, "https"}, - {"gzip", "gzip, deflate", true, "https"}, - - // Multiple request values, this means we will use multiple passes of compression, http - {"deflate", "deflate, gzip", true, "https"}, - {"gzip", "gzip, deflate", true, "https"}, + {"deflate", "deflate, gzip"}, + {"gzip", "gzip, deflate"}, // Multiple request values, this means we will use multiple passes of compression, https - {"deflate, gzip", "deflate, gzip", true, "https"}, - {"gzip, deflate", "gzip, deflate", true, "https"}, - - // x-gzip alias, http - {"deflate, gzip", "deflate, gzip", true, "https"}, - {"gzip, deflate", "gzip, deflate", true, "https"}, + {"deflate, gzip", "deflate, gzip"}, + {"gzip, deflate", "gzip, deflate"}, // x-gzip alias, https - {"x-gzip", "deflate, gzip", true, "https"}, - {"x-gzip", "gzip, deflate", true, "https"}, + {"x-gzip", "deflate, gzip"}, + {"x-gzip, deflate", "gzip, deflate"}, + {"deflate, x-gzip", "gzip, deflate"}, }; + + // For each scheme + Object[][] schemes = schemes(); + Object[][] result = new Object[compressOptions.length * 2][3]; + + int index = 0; + for (Object[] compressOption : compressOptions) { + for (Object[] scheme : schemes) { + result[index][0] = scheme[0]; // scheme + result[index][1] = compressOption[0]; // Content-Encoding + result[index][2] = compressOption[1]; // Accept-Encoding + index++; + } + } + + // scheme, Content-Encoding, Accept-Encoding + return result; } @Test(dataProvider = "chunkedSchemes") - public void requestedButNotAccepted(boolean chunked, String scheme) throws Exception { + public void requestedButNotAccepted(String scheme, boolean chunked) throws Exception { // Use case: setCompress(true), but the request does not contain the 'Accept-Encoding' header. // Result: no compression HTTPHandler handler = (req, res) -> { @@ -383,11 +383,19 @@ public void requestedButNotAccepted(boolean chunked, String scheme) throws Excep } @Test(dataProvider = "chunkedSchemes") - public void requestedButNotAccepted_unSupportedEncoding(boolean chunked, String scheme) throws Exception { + public void requestedButNotAccepted_unSupportedEncoding(String scheme, boolean chunked) throws Exception { // Use case: setCompress(true), and 'Accept-Encoding: br' which is valid, but not yet supported // Result: no compression HTTPHandler handler = (req, res) -> { res.setCompress(true); + + // Testing an indecisive user, can't make up their mind... this is allowed as long as you have not written ay bytes. + res.setCompress(true); + res.setCompress(false); + res.setCompress(true); + res.setCompress(false); + res.setCompress(true); + res.setHeader(Headers.ContentType, "text/plain"); res.setStatus(200); @@ -399,6 +407,14 @@ public void requestedButNotAccepted_unSupportedEncoding(boolean chunked, String try (InputStream is = Files.newInputStream(file)) { OutputStream outputStream = res.getOutputStream(); is.transferTo(outputStream); + + // Try to call setCompress, expect an exception - we cannot change modes once we've written to the OutputStream. + try { + res.setCompress(false); + fail("Expected setCompress(false) to fail hard!"); + } catch (IllegalStateException expected) { + } + outputStream.close(); } catch (IOException e) { throw new RuntimeException(e); @@ -418,4 +434,35 @@ public void requestedButNotAccepted_unSupportedEncoding(boolean chunked, String assertEquals(response.body(), Files.readString(file)); } } + + @Test(dataProvider = "schemes") + public void unsupportedContentType(String scheme) throws Exception { + // Use case: Tell the server we encoded the request using 'br' + HTTPHandler handler = (req, res) -> { + res.setHeader(Headers.ContentType, "text/plain"); + res.setStatus(200); + }; + + // The body isn't actually encoded, that is ok, we will validate that 'br' is not supported when we validate the preamble. + var bodyPublisher = BodyPublishers.ofInputStream(() -> new ByteArrayInputStream("Hello World".getBytes(StandardCharsets.UTF_8))); + + try (var client = makeClient(scheme, null); + var ignore = makeServer(scheme, handler, null).start()) { + + URI uri = makeURI(scheme, ""); + var response = client.send( + HttpRequest.newBuilder() + .uri(uri) + .header(Headers.ContentEncoding, "br") + .header(Headers.ContentType, "text/plain") + .POST(bodyPublisher) + .build(), + r -> BodySubscribers.ofString(StandardCharsets.UTF_8) + ); + + // Expect a 415 w/ an empty body response + assertEquals(response.statusCode(), 415); + assertEquals(response.body(), ""); + } + } } From c86206f2218d2bbd9a05895a5df2dbcea8e3e8f5 Mon Sep 17 00:00:00 2001 From: Daniel DeGroff Date: Wed, 29 Oct 2025 21:00:04 -0600 Subject: [PATCH 05/21] Working --- .../io/fusionauth/http/server/Configurable.java | 9 +++++++++ .../http/server/io/HTTPInputStream.java | 17 ++++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/main/java/io/fusionauth/http/server/Configurable.java b/src/main/java/io/fusionauth/http/server/Configurable.java index b745705e..21fab631 100644 --- a/src/main/java/io/fusionauth/http/server/Configurable.java +++ b/src/main/java/io/fusionauth/http/server/Configurable.java @@ -61,6 +61,15 @@ default T withChunkedBufferSize(int chunkedBufferSize) { /** * Sets the default compression behavior for the HTTP response. This behavior can be optionally set per response. See * {@link HTTPResponse#setCompress(boolean)}. Defaults to true. + *

+ * Set this configuration to true if you want to compress the response when the Accept-Encoding header is present. Set this + * configuration to false if you want to require the request handler to use {@link HTTPResponse#setCompress(boolean)} in + * order to compress the response. + *

+ * Regardless of this configuration, you always have the option to use {@link HTTPResponse#setCompress(boolean)} on a per-response basis + * as an override. + *

+ * When the request does not contain an Accept-Encoding the response will not be compressed regardless of this configuration. * * @param compressByDefault true if you want to compress by default, or false to not compress by default. * @return This. diff --git a/src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java b/src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java index 92b412d6..6c3219ae 100644 --- a/src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java +++ b/src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java @@ -135,6 +135,17 @@ public int read(byte[] b, int off, int len) throws IOException { return 0; } + // TODO : Re: Compression - fixedLength and Content-Length needs to account for the Content-Length referring to the compressed body. + + // Preamble, this could push back compressed bytes. + // Pushback > Throughput > Socket + // + // HTTPInputStream (this) -> Decompress > Pushback > Throughput > Socket + // HTTPInputStream (this) -> Decompress > Chunked > Pushback > Throughput > Socket + // + // Pushback doesn't care if the bytes are compressed or not, it is up to the caller? + // + // If this is a fixed length request, and we have less than or equal to 0 bytes remaining, return -1 boolean fixedLength = !request.isChunked(); if (fixedLength && bytesRemaining <= 0) { @@ -147,7 +158,11 @@ public int read(byte[] b, int off, int len) throws IOException { // When we have a fixed length request, read beyond the remainingBytes if possible. // - If we have read past the end of the current request, push those bytes back onto the InputStream. - int read = delegate.read(b, off, len); + // TODO : Can I optionally never override here? If I am fixed length, I could change len -> maxLen., and then never pushback. + // Would this help with compression? +// int maxLen = (int) Math.min(len, bytesRemaining); + int maxLen = len; + int read = delegate.read(b, off, maxLen); if (fixedLength && read > 0) { int extraBytes = (int) (read - bytesRemaining); if (extraBytes > 0) { From ccff2751499edeb1b8229e2c38ba3233c910702f Mon Sep 17 00:00:00 2001 From: Daniel DeGroff Date: Wed, 29 Oct 2025 22:37:33 -0600 Subject: [PATCH 06/21] Working --- .../java/io/fusionauth/http/server/io/HTTPInputStream.java | 7 +++---- .../io/fusionauth/http/server/io/HTTPOutputStream.java | 1 - 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java b/src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java index c6bc7737..0c7b4a97 100644 --- a/src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java +++ b/src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java @@ -20,8 +20,8 @@ import java.util.zip.GZIPInputStream; import java.util.zip.InflaterInputStream; -import io.fusionauth.http.HTTPValues.ContentEncodings; import io.fusionauth.http.ContentTooLargeException; +import io.fusionauth.http.HTTPValues.ContentEncodings; import io.fusionauth.http.io.ChunkedInputStream; import io.fusionauth.http.io.PushbackInputStream; import io.fusionauth.http.log.Logger; @@ -59,12 +59,12 @@ public class HTTPInputStream extends InputStream { private boolean closed; - private boolean initialized; - private InputStream delegate; private boolean drained; + private boolean initialized; + public HTTPInputStream(HTTPServerConfiguration configuration, HTTPRequest request, PushbackInputStream pushbackInputStream, int maximumContentLength) { this.logger = configuration.getLoggerFactory().getLogger(HTTPInputStream.class); @@ -175,7 +175,6 @@ public int read(byte[] b, int off, int len) throws IOException { // int maxLen = len; // int read = delegate.read(b, off, maxLen); - int reportBytesRead = read; if (fixedLength && read > 0) { int extraBytes = (int) (read - bytesRemaining); diff --git a/src/main/java/io/fusionauth/http/server/io/HTTPOutputStream.java b/src/main/java/io/fusionauth/http/server/io/HTTPOutputStream.java index 6c3e51ba..805df9bd 100644 --- a/src/main/java/io/fusionauth/http/server/io/HTTPOutputStream.java +++ b/src/main/java/io/fusionauth/http/server/io/HTTPOutputStream.java @@ -191,7 +191,6 @@ private void commit(boolean closing) throws IOException { response.setContentLength(0L); } else { // 204 status is specifically "No Content" so we shouldn't write the content-encoding and vary headers if the status is 204 - // TODO : Compress by default is on by default. But it looks like we don't actually compress unless you also send in an Accept-Encoding header? if (compress && !twoOhFour) { for (String encoding : acceptEncodings) { if (encoding.equalsIgnoreCase(ContentEncodings.Gzip)) { From dc6d8446c484305bb7729fe3f28d59bfc948bbdb Mon Sep 17 00:00:00 2001 From: Daniel DeGroff Date: Thu, 30 Oct 2025 00:49:39 -0600 Subject: [PATCH 07/21] working --- .../http/server/io/HTTPInputStream.java | 19 ++- .../io/fusionauth/http/BaseSocketTest.java | 3 +- .../java/io/fusionauth/http/BaseTest.java | 4 +- .../io/fusionauth/http/CompressionTest.java | 27 +++-- .../java/io/fusionauth/http/FormDataTest.java | 27 +++-- .../http/io/HTTPInputStreamTest.java | 111 +++++++++++++----- 6 files changed, 133 insertions(+), 58 deletions(-) diff --git a/src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java b/src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java index 0c7b4a97..65b9afb7 100644 --- a/src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java +++ b/src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java @@ -167,6 +167,16 @@ public int read(byte[] b, int off, int len) throws IOException { // - If we have read past the end of the current request, push those bytes back onto the InputStream. // - When a maximum content length has been specified, read at most one byte past the maximum. int maxReadLen = maximumContentLength == -1 ? len : Math.min(len, maximumContentLength - bytesRead + 1); + + // TODO : Hack - This makes compression work with fixed length requests + if (fixedLength) { + // TODO : Note : The len arg for an inflater stream is the number of un-compressed bytes. + // The returned number of bytes is the un-compressed bytes. So the problem here + // is that the bytesRemaining value we have is the compressed size of the payload so we can't + // us it to ensure we do not read past the current fixed length request. Sad. +// maxReadLen = Math.min(maxReadLen, (int) bytesRemaining); + } + int read = delegate.read(b, off, maxReadLen); // TODO : Can I optionally never override here? If I am fixed length, I could change len -> maxLen., and then never pushback. @@ -175,6 +185,10 @@ public int read(byte[] b, int off, int len) throws IOException { // int maxLen = len; // int read = delegate.read(b, off, maxLen); + // TODO : This is busted with compression. + // bytesRemaining is calculated based upon the Content-Length. + // But the bytes read from the InputStream will be the bytes read that are un-compressed. + // So we can't use the bytes read to calculate pushback. This has to go below the Decompression in the chain. int reportBytesRead = read; if (fixedLength && read > 0) { int extraBytes = (int) (read - bytesRemaining); @@ -254,8 +268,9 @@ private void initialize() throws IOException { // HTTPInputStream (this) > Pushback (delegate) > Throughput > Socket // HTTPInputStream (this) > Chunked (delegate) > Pushback > Throughput > Socket - // HTTPInputStream (this) > Pushback > Decompress > Throughput > Socket - // HTTPInputStream (this) > Pushback > Decompress > Chunked > Throughput > Socket + // The way it is currently coded + // HTTPInputStream (this) > Decompress > Pushback > Throughput > Socket + // HTTPInputStream (this) > Decompress > Chunked > Pushback > Throughput > Socket // TODO : Note I could leave this alone, but when we parse the header we can lower case these values and then remove the equalsIgnoreCase here? // Seems like ideally we would normalize them to lowercase earlier. diff --git a/src/test/java/io/fusionauth/http/BaseSocketTest.java b/src/test/java/io/fusionauth/http/BaseSocketTest.java index 5150d298..62647f11 100644 --- a/src/test/java/io/fusionauth/http/BaseSocketTest.java +++ b/src/test/java/io/fusionauth/http/BaseSocketTest.java @@ -66,7 +66,8 @@ private void assertResponse(String request, String chunkedExtension, String resp var body = bodyString.repeat(((requestBufferSize / bodyString.length())) * 2); if (request.contains("Transfer-Encoding: chunked")) { - body = chunkItUp(body, chunkedExtension); + // Chunk in 100 byte increments. Using a smaller chunk size to ensure we don't end up with a single chunk. + body = chunkItUp(body, 100, chunkedExtension); } request = request.replace("{body}", body); diff --git a/src/test/java/io/fusionauth/http/BaseTest.java b/src/test/java/io/fusionauth/http/BaseTest.java index 25ff4135..1d73e6fb 100644 --- a/src/test/java/io/fusionauth/http/BaseTest.java +++ b/src/test/java/io/fusionauth/http/BaseTest.java @@ -436,10 +436,8 @@ protected void assertHTTPResponseEquals(Socket socket, String expectedResponse) } } - protected String chunkItUp(String body, String chunkedExtension) { + protected String chunkItUp(String body, int chunkSize, String chunkedExtension) { List result = new ArrayList<>(); - // Chunk in 100 byte increments. Using a smaller chunk size to ensure we don't end up with a single chunk. - int chunkSize = 100; for (var i = 0; i < body.length(); i += chunkSize) { var endIndex = Math.min(i + chunkSize, body.length()); var chunk = body.substring(i, endIndex); diff --git a/src/test/java/io/fusionauth/http/CompressionTest.java b/src/test/java/io/fusionauth/http/CompressionTest.java index 58ef7ae0..6808e3f5 100644 --- a/src/test/java/io/fusionauth/http/CompressionTest.java +++ b/src/test/java/io/fusionauth/http/CompressionTest.java @@ -92,9 +92,11 @@ public void compress(String scheme, String contentEncoding, String acceptEncodin var requestEncodings = contentEncoding.toLowerCase().trim().split(","); for (String part : requestEncodings) { String encoding = part.trim(); - payload = encoding.equals(ContentEncodings.Deflate) - ? deflate(payload) - : gzip(payload); + if (encoding.equals(ContentEncodings.Deflate)) { + payload = deflate(payload); + } else if (encoding.equals(ContentEncodings.Gzip) || encoding.equals(ContentEncodings.XGzip)) { + payload = gzip(payload); + } } byte[] uncompressedBody = payload; @@ -102,9 +104,11 @@ public void compress(String scheme, String contentEncoding, String acceptEncodin // Sanity check on round trip compress/decompress for (int i = requestEncodings.length - 1; i >= 0; i--) { String encoding = requestEncodings[i].trim(); - uncompressedBody = encoding.equals(ContentEncodings.Deflate) - ? inflate(uncompressedBody) - : ungzip(uncompressedBody); + if (encoding.equals(ContentEncodings.Deflate)) { + uncompressedBody = inflate(uncompressedBody); + } else if (encoding.equals(ContentEncodings.Gzip) || encoding.equals(ContentEncodings.XGzip)) { + uncompressedBody = ungzip(uncompressedBody); + } } assertEquals(uncompressedBody, bodyBytes); @@ -140,17 +144,18 @@ public void compress(String scheme, String contentEncoding, String acceptEncodin assertNotNull(expectedResponseEncoding); - String result; + String result = null; InputStream responseInputStream = response.body(); if (expectedResponseEncoding.isEmpty()) { result = new String(responseInputStream.readAllBytes()); } else { assertEquals(response.headers().firstValue(Headers.ContentEncoding).orElse(null), expectedResponseEncoding); - result = new String( - expectedResponseEncoding.equals(ContentEncodings.Deflate) - ? new InflaterInputStream(responseInputStream).readAllBytes() - : new GZIPInputStream(responseInputStream).readAllBytes(), StandardCharsets.UTF_8); + if (expectedResponseEncoding.equals(ContentEncodings.Deflate)) { + result = new String(new InflaterInputStream(responseInputStream).readAllBytes()); + } else if (expectedResponseEncoding.equals(ContentEncodings.Gzip) || expectedResponseEncoding.equals(ContentEncodings.XGzip)) { + result = new String(new GZIPInputStream(responseInputStream).readAllBytes(), StandardCharsets.UTF_8); + } } assertEquals(result, Files.readString(file)); diff --git a/src/test/java/io/fusionauth/http/FormDataTest.java b/src/test/java/io/fusionauth/http/FormDataTest.java index 3c2dcac9..5ef32834 100644 --- a/src/test/java/io/fusionauth/http/FormDataTest.java +++ b/src/test/java/io/fusionauth/http/FormDataTest.java @@ -197,6 +197,18 @@ public Builder(String scheme) { this.scheme = scheme; } + public Builder assertOptionalExceptionOnWrite(Class clazz) { + // Note that this assertion really depends upon the system the test is run on, the size of the request, and the amount of data that can be cached. + // - So this is an optional assertion - if exception is not null, then we should be able to assert some attributes. + // - With the larger sizes this exception is mostly always thrown when running tests locally, but in GHA, it doesn't always occur. + if (thrownOnWrite != null) { + assertEquals(thrownOnWrite.getClass(), clazz); + assertEquals(thrownOnWrite.getMessage(), "Broken pipe"); + } + + return this; + } + public Builder expectNoExceptionOnWrite() { assertNull(thrownOnWrite); return this; @@ -272,7 +284,8 @@ public Builder expectResponse(String response) throws Exception { """; // Convert body to chunked - body = chunkItUp(body, null); + // - Using a small chunk to ensure we end up with more than one chunk. + body = chunkItUp(body, 100, null); } else { var contentLength = body.getBytes(StandardCharsets.UTF_8).length; if (contentLength > 0) { @@ -307,18 +320,6 @@ public Builder expectResponse(String response) throws Exception { return this; } - public Builder assertOptionalExceptionOnWrite(Class clazz) { - // Note that this assertion really depends upon the system the test is run on, the size of the request, and the amount of data that can be cached. - // - So this is an optional assertion - if exception is not null, then we should be able to assert some attributes. - // - With the larger sizes this exception is mostly always thrown when running tests locally, but in GHA, it doesn't always occur. - if (thrownOnWrite != null) { - assertEquals(thrownOnWrite.getClass(), clazz); - assertEquals(thrownOnWrite.getMessage(), "Broken pipe"); - } - - return this; - } - public Builder withBodyParameterCount(int bodyParameterCount) { this.bodyParameterCount = bodyParameterCount; return this; diff --git a/src/test/java/io/fusionauth/http/io/HTTPInputStreamTest.java b/src/test/java/io/fusionauth/http/io/HTTPInputStreamTest.java index 3dab2988..22c39446 100644 --- a/src/test/java/io/fusionauth/http/io/HTTPInputStreamTest.java +++ b/src/test/java/io/fusionauth/http/io/HTTPInputStreamTest.java @@ -16,63 +16,117 @@ package io.fusionauth.http.io; import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.nio.charset.StandardCharsets; +import io.fusionauth.http.BaseTest; +import io.fusionauth.http.HTTPValues.ContentEncodings; +import io.fusionauth.http.HTTPValues.Headers; import io.fusionauth.http.server.HTTPRequest; import io.fusionauth.http.server.HTTPServerConfiguration; import io.fusionauth.http.server.io.HTTPInputStream; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import static org.testng.Assert.assertEquals; /** * @author Daniel DeGroff */ -public class HTTPInputStreamTest { - @Test - public void read_chunked_withPushback() throws Exception { +public class HTTPInputStreamTest extends BaseTest { + @DataProvider(name = "contentEncoding") + public Object[][] contentEncoding() { + return new Object[][]{ + {""}, + {"gzip"}, + {"deflate"}, + {"gzip, deflate"}, + {"deflate, gzip"} + }; + } + + @Test(dataProvider = "contentEncoding") + public void read_chunked_withPushback(String contentEncoding) throws Exception { // Ensure that when we read a chunked encoded body that the InputStream returns the correct number of bytes read even when // we read past the end of the current request and use the PushbackInputStream. String content = "These pretzels are making me thirsty. These pretzels are making me thirsty. These pretzels are making me thirsty."; int contentLength = content.getBytes(StandardCharsets.UTF_8).length; - // Chunk the content - byte[] bytes = """ - 26\r - These pretzels are making me thirsty. \r - 26\r - These pretzels are making me thirsty. \r - 25\r - These pretzels are making me thirsty.\r - 0\r - \r - GET / HTTP/1.1\r - """.getBytes(); + // Chunk the content, add part of the next request + String chunked = chunkItUp(content, 38, null); + byte[] payload = chunked.getBytes(StandardCharsets.UTF_8); + + // Optionally compress the payload + if (!contentEncoding.isEmpty()) { + var requestEncodings = contentEncoding.toLowerCase().trim().split(","); + for (String part : requestEncodings) { + String encoding = part.trim(); + if (encoding.equals(ContentEncodings.Deflate)) { + payload = deflate(payload); + } else if (encoding.equals(ContentEncodings.Gzip)) { + payload = gzip(payload); + } + } + } + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + out.write(payload); + + // Add part of the next request + out.write("GET / HTTP/1.1\r\n".getBytes(StandardCharsets.UTF_8)); HTTPRequest request = new HTTPRequest(); - request.setHeader("Transfer-Encoding", "chunked"); + request.setHeader(Headers.ContentEncoding, contentEncoding); + request.setHeader(Headers.TransferEncoding, "chunked"); - assertReadWithPushback(bytes, content, contentLength, request); + byte[] bytes = out.toByteArray(); + assertReadWithPushback(bytes, content, contentLength, 16, request); } - @Test - public void read_fixedLength_withPushback() throws Exception { + @Test(dataProvider = "contentEncoding") + public void read_fixedLength_withPushback(String contentEncoding) throws Exception { // Ensure that when we read a fixed length body that the InputStream returns the correct number of bytes read even when // we read past the end of the current request and use the PushbackInputStream. - String content = "These pretzels are making me thirsty. These pretzels are making me thirsty. These pretzels are making me thirsty."; - int contentLength = content.getBytes(StandardCharsets.UTF_8).length; - // Fixed length body with the start of the next request in the buffer - byte[] bytes = (content + "GET / HTTP/1.1\r\n").getBytes(StandardCharsets.UTF_8); + String content = "These pretzels are making me thirsty. These pretzels are making me thirsty. These pretzels are making me thirsty."; + byte[] payload = content.getBytes(StandardCharsets.UTF_8); + int contentLength = payload.length; + + // Optionally compress the payload + if (!contentEncoding.isEmpty()) { + var requestEncodings = contentEncoding.toLowerCase().trim().split(","); + for (String part : requestEncodings) { + String encoding = part.trim(); + if (encoding.equals(ContentEncodings.Deflate)) { + payload = deflate(payload); + } else if (encoding.equals(ContentEncodings.Gzip)) { + payload = gzip(payload); + } + } + } + + // Content-Length must be the compressed length + int compressedLength = payload.length; + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + out.write(payload); + + // Add part of the next request + out.write("GET / HTTP/1.1\r\n".getBytes(StandardCharsets.UTF_8)); HTTPRequest request = new HTTPRequest(); - request.setHeader("Content-Length", contentLength + ""); + request.setHeader(Headers.ContentEncoding, contentEncoding); + request.setHeader(Headers.ContentLength, compressedLength + ""); - assertReadWithPushback(bytes, content, contentLength, request); + // body length is 113, when compressed it is 68 (gzip) or 78 (deflate) + // The number of bytes available is 129. + byte[] bytes = out.toByteArray(); + assertReadWithPushback(bytes, content, contentLength, 16, request); } - private void assertReadWithPushback(byte[] bytes, String content, int contentLength, HTTPRequest request) throws Exception { + private void assertReadWithPushback(byte[] bytes, String content, int contentLength, int pushedBack, HTTPRequest request) + throws Exception { int bytesAvailable = bytes.length; HTTPServerConfiguration configuration = new HTTPServerConfiguration().withRequestBufferSize(bytesAvailable + 100); @@ -83,6 +137,7 @@ private void assertReadWithPushback(byte[] bytes, String content, int contentLen byte[] buffer = new byte[configuration.getRequestBufferSize()]; int read = httpInputStream.read(buffer); + // TODO : Hmm.. with compression, this is returning the compressed bytes read instead of the un-compressed bytes read. WTF? assertEquals(read, contentLength); assertEquals(new String(buffer, 0, read), content); @@ -91,12 +146,12 @@ private void assertReadWithPushback(byte[] bytes, String content, int contentLen assertEquals(secondRead, -1); // We have 16 bytes left over - assertEquals(pushbackInputStream.getAvailableBufferedBytesRemaining(), 16); + assertEquals(pushbackInputStream.getAvailableBufferedBytesRemaining(), pushedBack); // Next read should start at the next request byte[] leftOverBuffer = new byte[100]; int leftOverRead = pushbackInputStream.read(leftOverBuffer); - assertEquals(leftOverRead, 16); + assertEquals(leftOverRead, pushedBack); assertEquals(new String(leftOverBuffer, 0, leftOverRead), "GET / HTTP/1.1\r\n"); } } From b4fd8c208fe81f913e4060b208f7bc829ce9be37 Mon Sep 17 00:00:00 2001 From: Daniel DeGroff Date: Thu, 30 Oct 2025 12:43:08 -0600 Subject: [PATCH 08/21] working --- .../http/io/FixedLengthInputStream.java | 68 ++++++++++++++++ .../http/server/io/HTTPInputStream.java | 78 +++---------------- .../http/io/HTTPInputStreamTest.java | 15 ++-- 3 files changed, 89 insertions(+), 72 deletions(-) create mode 100644 src/main/java/io/fusionauth/http/io/FixedLengthInputStream.java diff --git a/src/main/java/io/fusionauth/http/io/FixedLengthInputStream.java b/src/main/java/io/fusionauth/http/io/FixedLengthInputStream.java new file mode 100644 index 00000000..0b411a3a --- /dev/null +++ b/src/main/java/io/fusionauth/http/io/FixedLengthInputStream.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2025, FusionAuth, All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific + * language governing permissions and limitations under the License. + */ +package io.fusionauth.http.io; + +import java.io.IOException; +import java.io.InputStream; + +/** + * A filter InputStream that reads a fixed length body. + * + * @author Daniel DeGroff + */ +public class FixedLengthInputStream extends InputStream { + private final byte[] b1 = new byte[1]; + + private final PushbackInputStream delegate; + + private long bytesRemaining; + + public FixedLengthInputStream(PushbackInputStream delegate, long contentLength) { + this.delegate = delegate; + this.bytesRemaining = contentLength; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + if (bytesRemaining <= 0) { + return -1; + } + + int read = delegate.read(b, off, len); + int reportBytesRead = read; + if (read > 0) { + int extraBytes = (int) (read - bytesRemaining); + if (extraBytes > 0) { + reportBytesRead -= extraBytes; + delegate.push(b, (int) bytesRemaining, extraBytes); + } + + bytesRemaining -= reportBytesRead; + } + + return reportBytesRead; + } + + @Override + public int read() throws IOException { + var read = read(b1); + if (read <= 0) { + return read; + } + + return b1[0] & 0xFF; + } +} diff --git a/src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java b/src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java index 65b9afb7..df8d271c 100644 --- a/src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java +++ b/src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java @@ -23,6 +23,7 @@ import io.fusionauth.http.ContentTooLargeException; import io.fusionauth.http.HTTPValues.ContentEncodings; import io.fusionauth.http.io.ChunkedInputStream; +import io.fusionauth.http.io.FixedLengthInputStream; import io.fusionauth.http.io.PushbackInputStream; import io.fusionauth.http.log.Logger; import io.fusionauth.http.server.HTTPRequest; @@ -55,8 +56,6 @@ public class HTTPInputStream extends InputStream { private int bytesRead; - private long bytesRemaining; - private boolean closed; private InputStream delegate; @@ -75,11 +74,6 @@ public HTTPInputStream(HTTPServerConfiguration configuration, HTTPRequest reques this.chunkedBufferSize = configuration.getChunkedBufferSize(); this.maximumBytesToDrain = configuration.getMaxBytesToDrain(); this.maximumContentLength = maximumContentLength; - - // Start the countdown - if (request.getContentLength() != null) { - this.bytesRemaining = request.getContentLength(); - } } @Override @@ -142,23 +136,6 @@ public int read(byte[] b, int off, int len) throws IOException { return 0; } - // TODO : Re: Compression - fixedLength and Content-Length needs to account for the Content-Length referring to the compressed body. - - // Preamble, this could push back compressed bytes. - // Pushback > Throughput > Socket - // - // HTTPInputStream (this) -> Decompress > Pushback > Throughput > Socket - // HTTPInputStream (this) -> Decompress > Chunked > Pushback > Throughput > Socket - // - // Pushback doesn't care if the bytes are compressed or not, it is up to the caller? - // - - // If this is a fixed length request, and we have less than or equal to 0 bytes remaining, return -1 - boolean fixedLength = !request.isChunked(); - if (fixedLength && bytesRemaining <= 0) { - return -1; - } - if (!initialized) { initialize(); } @@ -167,41 +144,11 @@ public int read(byte[] b, int off, int len) throws IOException { // - If we have read past the end of the current request, push those bytes back onto the InputStream. // - When a maximum content length has been specified, read at most one byte past the maximum. int maxReadLen = maximumContentLength == -1 ? len : Math.min(len, maximumContentLength - bytesRead + 1); - - // TODO : Hack - This makes compression work with fixed length requests - if (fixedLength) { - // TODO : Note : The len arg for an inflater stream is the number of un-compressed bytes. - // The returned number of bytes is the un-compressed bytes. So the problem here - // is that the bytesRemaining value we have is the compressed size of the payload so we can't - // us it to ensure we do not read past the current fixed length request. Sad. -// maxReadLen = Math.min(maxReadLen, (int) bytesRemaining); - } - int read = delegate.read(b, off, maxReadLen); - - // TODO : Can I optionally never override here? If I am fixed length, I could change len -> maxLen., and then never pushback. - // Would this help with compression? -// int maxLen = (int) Math.min(len, bytesRemaining); -// int maxLen = len; -// int read = delegate.read(b, off, maxLen); - - // TODO : This is busted with compression. - // bytesRemaining is calculated based upon the Content-Length. - // But the bytes read from the InputStream will be the bytes read that are un-compressed. - // So we can't use the bytes read to calculate pushback. This has to go below the Decompression in the chain. - int reportBytesRead = read; - if (fixedLength && read > 0) { - int extraBytes = (int) (read - bytesRemaining); - if (extraBytes > 0) { - reportBytesRead -= extraBytes; - pushbackInputStream.push(b, (int) bytesRemaining, extraBytes); - } - - bytesRemaining -= reportBytesRead; + if (read > 0) { + bytesRead += read; } - bytesRead += reportBytesRead; - // Note that when the request is fixed length, we will have failed early during commit(). // - This will handle all requests that are not fixed length. if (maximumContentLength != -1 && bytesRead > maximumContentLength) { @@ -209,7 +156,7 @@ public int read(byte[] b, int off, int len) throws IOException { throw new ContentTooLargeException(maximumContentLength, detailedMessage); } - return reportBytesRead; + return read; } private void initialize() throws IOException { @@ -248,6 +195,7 @@ private void initialize() throws IOException { } } else if (contentLength != null) { logger.trace("Client indicated it was sending an entity-body in the request. Handling body using Content-Length header {}.", contentLength); + delegate = new FixedLengthInputStream(pushbackInputStream, contentLength); } else { logger.trace("Client indicated it was NOT sending an entity-body in the request"); } @@ -255,22 +203,18 @@ private void initialize() throws IOException { // If we have a maximumContentLength, and this is a fixed content length request, before we read any bytes, fail early. // For good measure do this last so if anyone downstream wants to read from the InputStream they could in theory because // we will have set up the InputStream. + // TODO : Compression : we may have to delete this code, or only enforce it when compression was not used. + // Content-Length represents the compressed size, not the uncompressed bytes. if (contentLength != null && maximumContentLength != -1 && contentLength > maximumContentLength) { String detailedMessage = "The maximum request size has been exceeded. The reported Content-Length is [" + contentLength + "] and the maximum request size is [" + maximumContentLength + "] bytes."; throw new ContentTooLargeException(maximumContentLength, detailedMessage); } - // Those who push back: - // HTTPInputStream when fixed - // ChunkedInputStream when chunked - // Preamble parser - - // HTTPInputStream (this) > Pushback (delegate) > Throughput > Socket - // HTTPInputStream (this) > Chunked (delegate) > Pushback > Throughput > Socket - // The way it is currently coded - // HTTPInputStream (this) > Decompress > Pushback > Throughput > Socket - // HTTPInputStream (this) > Decompress > Chunked > Pushback > Throughput > Socket + // HTTPInputStream (this) > Decompress > Fixed > Pushback > Throughput > Socket + + // HTTPInputStream (this) > Chunked > Pushback > Throughput > Socket + // > HTTPInputStream (this) > Decompress > Chunked > Pushback > Throughput > Socket // TODO : Note I could leave this alone, but when we parse the header we can lower case these values and then remove the equalsIgnoreCase here? // Seems like ideally we would normalize them to lowercase earlier. diff --git a/src/test/java/io/fusionauth/http/io/HTTPInputStreamTest.java b/src/test/java/io/fusionauth/http/io/HTTPInputStreamTest.java index 22c39446..e5335a84 100644 --- a/src/test/java/io/fusionauth/http/io/HTTPInputStreamTest.java +++ b/src/test/java/io/fusionauth/http/io/HTTPInputStreamTest.java @@ -50,11 +50,8 @@ public void read_chunked_withPushback(String contentEncoding) throws Exception { // we read past the end of the current request and use the PushbackInputStream. String content = "These pretzels are making me thirsty. These pretzels are making me thirsty. These pretzels are making me thirsty."; - int contentLength = content.getBytes(StandardCharsets.UTF_8).length; - - // Chunk the content, add part of the next request - String chunked = chunkItUp(content, 38, null); - byte[] payload = chunked.getBytes(StandardCharsets.UTF_8); + byte[] payload = content.getBytes(StandardCharsets.UTF_8); + int contentLength = payload.length; // Optionally compress the payload if (!contentEncoding.isEmpty()) { @@ -69,6 +66,14 @@ public void read_chunked_withPushback(String contentEncoding) throws Exception { } } + // Chunk the content, add part of the next request + var temp1 = payload; + var temp2 = new String(payload, StandardCharsets.UTF_8).getBytes(StandardCharsets.UTF_8); + assertEquals(temp1, temp2); + + String chunked = chunkItUp(new String(payload, StandardCharsets.UTF_8), 38, null); + payload = chunked.getBytes(StandardCharsets.UTF_8); + ByteArrayOutputStream out = new ByteArrayOutputStream(); out.write(payload); From 17715d4f91f55ec5cfe416cd259b45c50304ebc5 Mon Sep 17 00:00:00 2001 From: Daniel DeGroff Date: Thu, 30 Oct 2025 14:09:38 -0600 Subject: [PATCH 09/21] format --- .../java/io/fusionauth/http/FormDataTest.java | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/test/java/io/fusionauth/http/FormDataTest.java b/src/test/java/io/fusionauth/http/FormDataTest.java index 8f254ba7..9984f027 100644 --- a/src/test/java/io/fusionauth/http/FormDataTest.java +++ b/src/test/java/io/fusionauth/http/FormDataTest.java @@ -199,6 +199,17 @@ public Builder(String scheme) { this.scheme = scheme; } + public Builder assertOptionalExceptionOnWrite(Class clazz) { + // Note that this assertion really depends upon the system the test is run on, the size of the request, and the amount of data that can be cached. + // - So this is an optional assertion - if exception is not null, then we should be able to assert some attributes. + // - With the larger sizes this exception is mostly always thrown when running tests locally, but in GHA, it doesn't always occur. + if (thrownOnWrite != null) { + assertEquals(thrownOnWrite.getClass(), clazz); + } + + return this; + } + public Builder expectNoExceptionOnWrite() { assertNull(thrownOnWrite); return this; @@ -310,17 +321,6 @@ public Builder expectResponse(String response) throws Exception { return this; } - public Builder assertOptionalExceptionOnWrite(Class clazz) { - // Note that this assertion really depends upon the system the test is run on, the size of the request, and the amount of data that can be cached. - // - So this is an optional assertion - if exception is not null, then we should be able to assert some attributes. - // - With the larger sizes this exception is mostly always thrown when running tests locally, but in GHA, it doesn't always occur. - if (thrownOnWrite != null) { - assertEquals(thrownOnWrite.getClass(), clazz); - } - - return this; - } - public Builder withBodyParameterCount(int bodyParameterCount) { this.bodyParameterCount = bodyParameterCount; return this; From bf41b95555af868fa8dc9f9cf5f8452d8e8e0493 Mon Sep 17 00:00:00 2001 From: Daniel DeGroff Date: Thu, 30 Oct 2025 14:37:18 -0600 Subject: [PATCH 10/21] cleanup, testing. --- .../http/server/io/HTTPInputStream.java | 29 +++++-------------- .../io/fusionauth/http/BaseSocketTest.java | 2 +- .../java/io/fusionauth/http/BaseTest.java | 28 +++++++++--------- .../java/io/fusionauth/http/FormDataTest.java | 2 +- .../http/io/HTTPInputStreamTest.java | 7 +---- 5 files changed, 26 insertions(+), 42 deletions(-) diff --git a/src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java b/src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java index df8d271c..a5deb80a 100644 --- a/src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java +++ b/src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java @@ -168,21 +168,6 @@ private void initialize() throws IOException { Long contentLength = request.getContentLength(); boolean hasBody = (contentLength != null && contentLength > 0) || request.isChunked(); - // Request order of operations: - // - "hello world" -> compress -> chunked. - - // Current: - // Chunked: - // HTTPInputStream > Chunked > Pushback > Throughput > Socket - // Fixed: - // HTTPInputStream > Pushback > Throughput > Socket - - // New: - // Chunked - // HTTPInputStream > Chunked > Pushback > Decompress > Throughput > Socket - // Fixed - // HTTPInputStream > Pushback > Decompress > Throughput > Socket - // Note that isChunked() should take precedence over the fact that we have a Content-Length. // - The client should not send both, but in the case they are both present we ignore Content-Length if (!hasBody) { @@ -210,14 +195,16 @@ private void initialize() throws IOException { throw new ContentTooLargeException(maximumContentLength, detailedMessage); } - // The way it is currently coded - // HTTPInputStream (this) > Decompress > Fixed > Pushback > Throughput > Socket + // Current state + // Chunked HTTPInputStream (this) > Chunked > Pushback > Throughput > Socket + // Fixed length HTTPInputStream (this) > Fixed > Pushback > Throughput > Socket - // HTTPInputStream (this) > Chunked > Pushback > Throughput > Socket - // > HTTPInputStream (this) > Decompress > Chunked > Pushback > Throughput > Socket + // When decompressing, the result will be something like this: + // Chunked HTTPInputStream (this) > Decompress > Chunked > Pushback > Throughput > Socket + // Fixed length HTTPInputStream (this) > Decompress> Fixed > Pushback > Throughput > Socket + // + // You may have one or more Decompress InputStreams in the above diagram. - // TODO : Note I could leave this alone, but when we parse the header we can lower case these values and then remove the equalsIgnoreCase here? - // Seems like ideally we would normalize them to lowercase earlier. if (hasBody) { // The request may contain more than one value, apply in reverse order. // - These are both using the default 512 buffer size. diff --git a/src/test/java/io/fusionauth/http/BaseSocketTest.java b/src/test/java/io/fusionauth/http/BaseSocketTest.java index 62647f11..0109f6d0 100644 --- a/src/test/java/io/fusionauth/http/BaseSocketTest.java +++ b/src/test/java/io/fusionauth/http/BaseSocketTest.java @@ -67,7 +67,7 @@ private void assertResponse(String request, String chunkedExtension, String resp if (request.contains("Transfer-Encoding: chunked")) { // Chunk in 100 byte increments. Using a smaller chunk size to ensure we don't end up with a single chunk. - body = chunkItUp(body, 100, chunkedExtension); + body = new String(chunkEncoded(body.getBytes(StandardCharsets.UTF_8), 100, chunkedExtension)); } request = request.replace("{body}", body); diff --git a/src/test/java/io/fusionauth/http/BaseTest.java b/src/test/java/io/fusionauth/http/BaseTest.java index 1d73e6fb..bde18f95 100644 --- a/src/test/java/io/fusionauth/http/BaseTest.java +++ b/src/test/java/io/fusionauth/http/BaseTest.java @@ -49,10 +49,8 @@ import java.time.Instant; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; -import java.util.ArrayList; import java.util.Arrays; import java.util.Date; -import java.util.List; import java.util.Map; import java.util.UUID; import java.util.stream.Collectors; @@ -436,25 +434,29 @@ protected void assertHTTPResponseEquals(Socket socket, String expectedResponse) } } - protected String chunkItUp(String body, int chunkSize, String chunkedExtension) { - List result = new ArrayList<>(); - for (var i = 0; i < body.length(); i += chunkSize) { - var endIndex = Math.min(i + chunkSize, body.length()); - var chunk = body.substring(i, endIndex); - var chunkLength = chunk.getBytes(StandardCharsets.UTF_8).length; + protected byte[] chunkEncoded(byte[] bytes, int chunkSize, String chunkedExtension) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + for (var i = 0; i < bytes.length; i += chunkSize) { + var endIndex = Math.min(i + chunkSize, bytes.length); + + var chunk = Arrays.copyOfRange(bytes, i, endIndex); + var chunkLength = chunk.length; String hex = Integer.toHexString(chunkLength); - result.add(hex); + out.write(hex.getBytes(StandardCharsets.UTF_8)); if (chunkedExtension != null) { - result.add(chunkedExtension); + out.write(chunkedExtension.getBytes(StandardCharsets.UTF_8)); } - result.add(("\r\n" + chunk + "\r\n")); + out.write("\r\n".getBytes(StandardCharsets.UTF_8)); + out.write(chunk); + out.write("\r\n".getBytes(StandardCharsets.UTF_8)); } - result.add(("0\r\n\r\n")); - return String.join("", result); + + out.write(("0\r\n\r\n".getBytes(StandardCharsets.UTF_8))); + return out.toByteArray(); } protected byte[] deflate(byte[] bytes) throws Exception { diff --git a/src/test/java/io/fusionauth/http/FormDataTest.java b/src/test/java/io/fusionauth/http/FormDataTest.java index 9984f027..bc9909f5 100644 --- a/src/test/java/io/fusionauth/http/FormDataTest.java +++ b/src/test/java/io/fusionauth/http/FormDataTest.java @@ -286,7 +286,7 @@ public Builder expectResponse(String response) throws Exception { // Convert body to chunked // - Using a small chunk to ensure we end up with more than one chunk. - body = chunkItUp(body, 100, null); + body = new String(chunkEncoded(body.getBytes(StandardCharsets.UTF_8), 100, null)); } else { var contentLength = body.getBytes(StandardCharsets.UTF_8).length; if (contentLength > 0) { diff --git a/src/test/java/io/fusionauth/http/io/HTTPInputStreamTest.java b/src/test/java/io/fusionauth/http/io/HTTPInputStreamTest.java index e5335a84..b8c36fc0 100644 --- a/src/test/java/io/fusionauth/http/io/HTTPInputStreamTest.java +++ b/src/test/java/io/fusionauth/http/io/HTTPInputStreamTest.java @@ -67,12 +67,7 @@ public void read_chunked_withPushback(String contentEncoding) throws Exception { } // Chunk the content, add part of the next request - var temp1 = payload; - var temp2 = new String(payload, StandardCharsets.UTF_8).getBytes(StandardCharsets.UTF_8); - assertEquals(temp1, temp2); - - String chunked = chunkItUp(new String(payload, StandardCharsets.UTF_8), 38, null); - payload = chunked.getBytes(StandardCharsets.UTF_8); + payload = chunkEncoded(payload, 38, null); ByteArrayOutputStream out = new ByteArrayOutputStream(); out.write(payload); From 6ec7fefb25a8cc94c53d1464e4c233ee20c7c811 Mon Sep 17 00:00:00 2001 From: Daniel DeGroff Date: Thu, 30 Oct 2025 16:14:50 -0600 Subject: [PATCH 11/21] cleanup, testing. --- .../http/server/io/HTTPInputStream.java | 46 ++++++------------- 1 file changed, 15 insertions(+), 31 deletions(-) diff --git a/src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java b/src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java index a5deb80a..5a482798 100644 --- a/src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java +++ b/src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java @@ -31,9 +31,9 @@ import io.fusionauth.http.server.Instrumenter; /** - * An InputStream that handles the HTTP body, including body bytes that were read while the preamble was processed. This class also handles - * chunked bodies by using a delegate InputStream that wraps the original source of the body bytes. The {@link ChunkedInputStream} is the - * delegate that this class leverages for chunking. + * An InputStream intended to be read the HTTP request body. + *

+ * This will handle fixed length requests, chunked requests as well as decompression if necessary. * * @author Brian Pontarelli */ @@ -140,17 +140,14 @@ public int read(byte[] b, int off, int len) throws IOException { initialize(); } - // When we have a fixed length request, read beyond the remainingBytes if possible. - // - If we have read past the end of the current request, push those bytes back onto the InputStream. - // - When a maximum content length has been specified, read at most one byte past the maximum. + // When a maximum content length has been specified, read at most one byte past the maximum. int maxReadLen = maximumContentLength == -1 ? len : Math.min(len, maximumContentLength - bytesRead + 1); int read = delegate.read(b, off, maxReadLen); if (read > 0) { bytesRead += read; } - // Note that when the request is fixed length, we will have failed early during commit(). - // - This will handle all requests that are not fixed length. + // Throw an exception once we have read past the maximum configured content length, if (maximumContentLength != -1 && bytesRead > maximumContentLength) { String detailedMessage = "The maximum request size has been exceeded. The maximum request size is [" + maximumContentLength + "] bytes."; throw new ContentTooLargeException(maximumContentLength, detailedMessage); @@ -167,9 +164,6 @@ private void initialize() throws IOException { // In practice, we will remove the Content-Length header when sent in addition to Transfer-Encoding. See HTTPWorker.validatePreamble. Long contentLength = request.getContentLength(); boolean hasBody = (contentLength != null && contentLength > 0) || request.isChunked(); - - // Note that isChunked() should take precedence over the fact that we have a Content-Length. - // - The client should not send both, but in the case they are both present we ignore Content-Length if (!hasBody) { delegate = InputStream.nullInputStream(); } else if (request.isChunked()) { @@ -185,26 +179,7 @@ private void initialize() throws IOException { logger.trace("Client indicated it was NOT sending an entity-body in the request"); } - // If we have a maximumContentLength, and this is a fixed content length request, before we read any bytes, fail early. - // For good measure do this last so if anyone downstream wants to read from the InputStream they could in theory because - // we will have set up the InputStream. - // TODO : Compression : we may have to delete this code, or only enforce it when compression was not used. - // Content-Length represents the compressed size, not the uncompressed bytes. - if (contentLength != null && maximumContentLength != -1 && contentLength > maximumContentLength) { - String detailedMessage = "The maximum request size has been exceeded. The reported Content-Length is [" + contentLength + "] and the maximum request size is [" + maximumContentLength + "] bytes."; - throw new ContentTooLargeException(maximumContentLength, detailedMessage); - } - - // Current state - // Chunked HTTPInputStream (this) > Chunked > Pushback > Throughput > Socket - // Fixed length HTTPInputStream (this) > Fixed > Pushback > Throughput > Socket - - // When decompressing, the result will be something like this: - // Chunked HTTPInputStream (this) > Decompress > Chunked > Pushback > Throughput > Socket - // Fixed length HTTPInputStream (this) > Decompress> Fixed > Pushback > Throughput > Socket - // - // You may have one or more Decompress InputStreams in the above diagram. - + // Now that we have the InputStream set up to read the body, handle decompression. if (hasBody) { // The request may contain more than one value, apply in reverse order. // - These are both using the default 512 buffer size. @@ -216,5 +191,14 @@ private void initialize() throws IOException { } } } + + // If we have a fixed length request that is reporting a contentLength larger than the configured maximum, fail earl. + // - Do this last so if anyone downstream wants to read from the InputStream it would work. + // - Note that it is possible that the body is compressed which would mean the contentLength represents the compressed value. + // But when we decompress the bytes the result will be larger than the reported contentLength, so we can safely throw this exception. + if (contentLength != null && maximumContentLength != -1 && contentLength > maximumContentLength) { + String detailedMessage = "The maximum request size has been exceeded. The reported Content-Length is [" + contentLength + "] and the maximum request size is [" + maximumContentLength + "] bytes."; + throw new ContentTooLargeException(maximumContentLength, detailedMessage); + } } } From 89362fe770c9cf0c8a62f2db044d17405990090f Mon Sep 17 00:00:00 2001 From: Daniel DeGroff Date: Thu, 30 Oct 2025 17:04:58 -0600 Subject: [PATCH 12/21] Tests --- .../java/io/fusionauth/http/BaseTest.java | 26 ++++++++ .../http/io/HTTPInputStreamTest.java | 39 ++---------- .../fusionauth/http/util/HTTPToolsTest.java | 59 ++++++++++++------- 3 files changed, 69 insertions(+), 55 deletions(-) diff --git a/src/test/java/io/fusionauth/http/BaseTest.java b/src/test/java/io/fusionauth/http/BaseTest.java index bde18f95..7caad08c 100644 --- a/src/test/java/io/fusionauth/http/BaseTest.java +++ b/src/test/java/io/fusionauth/http/BaseTest.java @@ -60,6 +60,7 @@ import java.util.zip.InflaterInputStream; import io.fusionauth.http.HTTPValues.Connections; +import io.fusionauth.http.HTTPValues.ContentEncodings; import io.fusionauth.http.log.FileLogger; import io.fusionauth.http.log.FileLoggerFactory; import io.fusionauth.http.log.Level; @@ -104,7 +105,21 @@ * @author Brian Pontarelli */ public abstract class BaseTest { + protected byte[] compressUsingContentEncoding(byte[] bytes, String contentEncoding) throws Exception { + if (!contentEncoding.isEmpty()) { + var requestEncodings = contentEncoding.toLowerCase().trim().split(","); + for (String part : requestEncodings) { + String encoding = part.trim(); + if (encoding.equals(ContentEncodings.Deflate)) { + bytes = deflate(bytes); + } else if (encoding.equals(ContentEncodings.Gzip)) { + bytes = gzip(bytes); + } + } + } + return bytes; + } /** * This timeout is used for the HttpClient during each test. If you are in a debugger, you will need to change this timeout to be much * larger, otherwise, the client might truncate the request to the server. @@ -248,6 +263,17 @@ public Object[][] connections() { }; } + @DataProvider(name = "contentEncoding") + public Object[][] contentEncoding() { + return new Object[][]{ + {""}, + {"gzip"}, + {"deflate"}, + {"gzip, deflate"}, + {"deflate, gzip"} + }; + } + @AfterMethod public void flush() { FileLogger fl = (FileLogger) FileLoggerFactory.FACTORY.getLogger(BaseTest.class); diff --git a/src/test/java/io/fusionauth/http/io/HTTPInputStreamTest.java b/src/test/java/io/fusionauth/http/io/HTTPInputStreamTest.java index b8c36fc0..0125e6b9 100644 --- a/src/test/java/io/fusionauth/http/io/HTTPInputStreamTest.java +++ b/src/test/java/io/fusionauth/http/io/HTTPInputStreamTest.java @@ -33,17 +33,6 @@ * @author Daniel DeGroff */ public class HTTPInputStreamTest extends BaseTest { - @DataProvider(name = "contentEncoding") - public Object[][] contentEncoding() { - return new Object[][]{ - {""}, - {"gzip"}, - {"deflate"}, - {"gzip, deflate"}, - {"deflate, gzip"} - }; - } - @Test(dataProvider = "contentEncoding") public void read_chunked_withPushback(String contentEncoding) throws Exception { // Ensure that when we read a chunked encoded body that the InputStream returns the correct number of bytes read even when @@ -53,18 +42,8 @@ public void read_chunked_withPushback(String contentEncoding) throws Exception { byte[] payload = content.getBytes(StandardCharsets.UTF_8); int contentLength = payload.length; - // Optionally compress the payload - if (!contentEncoding.isEmpty()) { - var requestEncodings = contentEncoding.toLowerCase().trim().split(","); - for (String part : requestEncodings) { - String encoding = part.trim(); - if (encoding.equals(ContentEncodings.Deflate)) { - payload = deflate(payload); - } else if (encoding.equals(ContentEncodings.Gzip)) { - payload = gzip(payload); - } - } - } + // Optionally compress + payload = compressUsingContentEncoding(payload, contentEncoding); // Chunk the content, add part of the next request payload = chunkEncoded(payload, 38, null); @@ -93,18 +72,8 @@ public void read_fixedLength_withPushback(String contentEncoding) throws Excepti byte[] payload = content.getBytes(StandardCharsets.UTF_8); int contentLength = payload.length; - // Optionally compress the payload - if (!contentEncoding.isEmpty()) { - var requestEncodings = contentEncoding.toLowerCase().trim().split(","); - for (String part : requestEncodings) { - String encoding = part.trim(); - if (encoding.equals(ContentEncodings.Deflate)) { - payload = deflate(payload); - } else if (encoding.equals(ContentEncodings.Gzip)) { - payload = gzip(payload); - } - } - } + // Optionally compress + payload = compressUsingContentEncoding(payload, contentEncoding); // Content-Length must be the compressed length int compressedLength = payload.length; diff --git a/src/test/java/io/fusionauth/http/util/HTTPToolsTest.java b/src/test/java/io/fusionauth/http/util/HTTPToolsTest.java index 50f2e479..bf24f5b8 100644 --- a/src/test/java/io/fusionauth/http/util/HTTPToolsTest.java +++ b/src/test/java/io/fusionauth/http/util/HTTPToolsTest.java @@ -16,6 +16,7 @@ package io.fusionauth.http.util; import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.net.URLEncoder; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; @@ -26,6 +27,7 @@ import java.util.Map; import com.inversoft.json.ToString; +import io.fusionauth.http.BaseTest; import io.fusionauth.http.HTTPMethod; import io.fusionauth.http.io.PushbackInputStream; import io.fusionauth.http.server.HTTPRequest; @@ -42,7 +44,7 @@ * @author Brian Pontarelli */ @Test -public class HTTPToolsTest { +public class HTTPToolsTest extends BaseTest { @Test public void getMaxRequestBodySize() { var configuration = Map.of( @@ -183,16 +185,18 @@ public void parseHeaderValue() { assertEquals(HTTPTools.parseHeaderValue("value; f= f"), new HeaderValue("value", Map.of("f", " f"))); } - @Test - public void parsePreamble() throws Exception { + @Test(dataProvider = "contentEncoding") + public void parsePreamble(String contentEncoding) throws Exception { // Ensure that we can correctly read the preamble when the InputStream contains the next request. + // - Optionally compress the body to ensure we can push back bytes on the preamble read w/out hosing it up. - //noinspection ExtractMethodRecommender - String request = """ + ByteArrayOutputStream out = new ByteArrayOutputStream(); + String header = """ GET / HTTP/1.1\r Host: localhost:42\r Connection: close\r - Content-Length: 113\r + Content-Encoding: {contentEncoding}\r + Content-Length: {contentLength}\r Header1: Value1\r Header2: Value2\r Header3: Value3\r @@ -204,17 +208,31 @@ public void parsePreamble() throws Exception { Header9: Value9\r Header10: Value10\r \r - These pretzels are making me thirsty. These pretzels are making me thirsty. These pretzels are making me thirsty.GET / HTTP/1.1\r - """; + """.replace("{contentEncoding}", contentEncoding); + + // Now add the body, optionally compressed + String body = "These pretzels are making me thirsty. These pretzels are making me thirsty. These pretzels are making me thirsty."; + byte[] bodyBytes = body.getBytes(StandardCharsets.UTF_8); + int uncompressedBodyLength = bodyBytes.length; + bodyBytes = compressUsingContentEncoding(bodyBytes, contentEncoding); + + header = header.replace("{contentLength}", bodyBytes.length + ""); + out.write(header.getBytes(StandardCharsets.UTF_8)); + out.write(bodyBytes); + + // Now add part of the next request + String nextRequest = "GET / HTTP/1.1\n\r"; + byte[] nextRequestBytes = nextRequest.getBytes(StandardCharsets.UTF_8); + out.write(nextRequestBytes); // Fixed length body with the start of the next request in the buffer - byte[] bytes = request.getBytes(StandardCharsets.UTF_8); - int bytesAvailable = bytes.length; + byte[] payload = out.toByteArray(); + int bytesAvailable = payload.length; // Ensure the request buffer size will contain the entire request. HTTPServerConfiguration configuration = new HTTPServerConfiguration().withRequestBufferSize(bytesAvailable + 100); - ByteArrayInputStream is = new ByteArrayInputStream(bytes); + ByteArrayInputStream is = new ByteArrayInputStream(payload); PushbackInputStream pushbackInputStream = new PushbackInputStream(is, null); HTTPRequest httpRequest = new HTTPRequest(); @@ -229,37 +247,38 @@ public void parsePreamble() throws Exception { assertEquals(httpRequest.getMethod(), HTTPMethod.GET); assertEquals(httpRequest.getHost(), "localhost"); assertEquals(httpRequest.getPort(), 42); - assertEquals(httpRequest.getContentLength(), 113); - assertEquals(httpRequest.getHeaders().size(), 13); - assertEquals(httpRequest.getHeader("Content-Length"), "113"); + assertEquals(httpRequest.getContentLength(), bodyBytes.length); + assertEquals(httpRequest.getHeaders().size(), contentEncoding.isEmpty() ? 13 : 14); + assertEquals(httpRequest.getHeader("Content-Encoding"), contentEncoding.isEmpty() ? null : contentEncoding); + assertEquals(httpRequest.getHeader("Content-Length"), bodyBytes.length + ""); assertEquals(httpRequest.getHeader("Connection"), "close"); assertEquals(httpRequest.getHeader("Host"), "localhost:42"); for (int i = 1; i <= 10; i++) { assertEquals(httpRequest.getHeader("Header" + i), "Value" + i); } - // Expect 129 bytes left over which is 113 for the body + 16 from the next request - assertEquals(pushbackInputStream.getAvailableBufferedBytesRemaining(), 113 + 16); + // Expect that the body bytes and the next request have been pushed into the pushback buffer + assertEquals(pushbackInputStream.getAvailableBufferedBytesRemaining(), bodyBytes.length + nextRequestBytes.length); // Read the remaining bytes for this request, we should still have some left over. HTTPInputStream httpInputStream = new HTTPInputStream(configuration, httpRequest, pushbackInputStream, -1); byte[] buffer = new byte[1024]; int read = httpInputStream.read(buffer); - assertEquals(read, 113); + assertEquals(read, uncompressedBodyLength); // Another read should return -1 because we are at the end of this request. int nextRead = httpInputStream.read(buffer); assertEquals(nextRead, -1); // The next read from the pushback which will be used by the next request should return the remaining bytes. - assertEquals(pushbackInputStream.getAvailableBufferedBytesRemaining(), 16); + assertEquals(pushbackInputStream.getAvailableBufferedBytesRemaining(), nextRequestBytes.length); int nextRequestRead = pushbackInputStream.read(buffer); - assertEquals(nextRequestRead, 16); + // Expect that we are able to read the bytes for the next request header + assertEquals(nextRequestRead, nextRequestBytes.length); } private void assertEncodedData(String actualValue, String expectedValue, Charset charset) { assertEncodedData(actualValue, expectedValue, charset, charset); - } private void assertEncodedData(String actualValue, String expectedValue, Charset encodingCharset, Charset decodingCharset) { From afc52e1558c427c530440cbd838ba1ab7054c40f Mon Sep 17 00:00:00 2001 From: Daniel DeGroff Date: Sat, 1 Nov 2025 12:14:18 -0600 Subject: [PATCH 13/21] Cleanup --- .../fusionauth/http/server/HTTPRequest.java | 8 ++- .../http/server/internal/HTTPWorker.java | 10 +--- .../http/server/io/HTTPInputStream.java | 60 ++++++++++--------- .../io/fusionauth/http/BaseSocketTest.java | 2 +- .../java/io/fusionauth/http/BaseTest.java | 43 ++++++------- .../java/io/fusionauth/http/FormDataTest.java | 2 +- .../http/io/HTTPInputStreamTest.java | 30 ++++++---- 7 files changed, 83 insertions(+), 72 deletions(-) diff --git a/src/main/java/io/fusionauth/http/server/HTTPRequest.java b/src/main/java/io/fusionauth/http/server/HTTPRequest.java index 589b2192..42310d73 100644 --- a/src/main/java/io/fusionauth/http/server/HTTPRequest.java +++ b/src/main/java/io/fusionauth/http/server/HTTPRequest.java @@ -626,12 +626,16 @@ public void setURLParameters(Map> parameters) { * {@code Content-Length} header was provided. */ public boolean hasBody() { + if (isChunked()) { + return true; + } + Long contentLength = getContentLength(); - return isChunked() || (contentLength != null && contentLength > 0); + return contentLength != null && contentLength > 0; } public boolean isChunked() { - return getTransferEncoding() != null && getTransferEncoding().equalsIgnoreCase(TransferEncodings.Chunked); + return TransferEncodings.Chunked.equalsIgnoreCase(getTransferEncoding()); } /** diff --git a/src/main/java/io/fusionauth/http/server/internal/HTTPWorker.java b/src/main/java/io/fusionauth/http/server/internal/HTTPWorker.java index 33627cee..b77e1da9 100644 --- a/src/main/java/io/fusionauth/http/server/internal/HTTPWorker.java +++ b/src/main/java/io/fusionauth/http/server/internal/HTTPWorker.java @@ -418,7 +418,6 @@ private Integer validatePreamble(HTTPRequest request) { // However, as long as we ignore Content-Length we should be ok. Earlier specs indicate Transfer-Encoding should take precedence, // later specs imply it is an error. Seems ok to allow it and just ignore it. if (request.getHeader(Headers.TransferEncoding) == null) { - var contentLength = request.getContentLength(); var requestedContentLengthHeaders = request.getHeaders(Headers.ContentLength); if (requestedContentLengthHeaders != null) { if (requestedContentLengthHeaders.size() != 1) { @@ -430,6 +429,7 @@ private Integer validatePreamble(HTTPRequest request) { return Status.BadRequest; } + var contentLength = request.getContentLength(); if (contentLength == null || contentLength < 0) { if (debugEnabled) { logger.debug("Invalid request. The Content-Length must be >= 0 and <= 9,223,372,036,854,775,807. [{}]", requestedContentLengthHeaders.getFirst()); @@ -445,13 +445,9 @@ private Integer validatePreamble(HTTPRequest request) { request.removeHeader(Headers.ContentLength); } - // Content-Encoding + // Validate Content-Encoding, we currently support deflate and gzip. + // - If we see anything else we should fail, we will be unable to handle the request. var contentEncodings = request.getContentEncodings(); - // TODO : If provided, I think we can safely remove the Content-Length header if present? - // Although, I suppose as long as we decompress early, we should still be able to use the Content-Length header as long as - // it represents the un-compressed payload. - // Maybe write a test to prove that we can compress with a fixed-length w/ a Content-Length header? - // Current support is for gzip and deflate. for (var encoding : contentEncodings) { if (!encoding.equalsIgnoreCase(ContentEncodings.Gzip) && !encoding.equalsIgnoreCase(ContentEncodings.Deflate)) { // Note that while we do not expect multiple Content-Encoding headers, the last one will be used. For good measure, diff --git a/src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java b/src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java index 5a482798..ee5f3835 100644 --- a/src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java +++ b/src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java @@ -159,28 +159,24 @@ public int read(byte[] b, int off, int len) throws IOException { private void initialize() throws IOException { initialized = true; - // Note that isChunked() should take precedence over the fact that we have a Content-Length. - // - The client should not send both, but in the case they are both present we ignore Content-Length - // In practice, we will remove the Content-Length header when sent in addition to Transfer-Encoding. See HTTPWorker.validatePreamble. - Long contentLength = request.getContentLength(); - boolean hasBody = (contentLength != null && contentLength > 0) || request.isChunked(); - if (!hasBody) { - delegate = InputStream.nullInputStream(); - } else if (request.isChunked()) { - logger.trace("Client indicated it was sending an entity-body in the request. Handling body using chunked encoding."); - delegate = new ChunkedInputStream(pushbackInputStream, chunkedBufferSize); - if (instrumenter != null) { - instrumenter.chunkedRequest(); + // hasBody means we are either using chunked transfer encoding or we have a non-zero Content-Length. + boolean hasBody = request.hasBody(); + if (hasBody) { + Long contentLength = request.getContentLength(); + // Transfer-Encoding always takes precedence over Content-Length. In practice if they were to both be present on + // the request we would have removed Content-Length during validation to remove ambiguity. See HTTPWorker.validatePreamble. + if (request.isChunked()) { + logger.trace("Client indicated it was sending an entity-body in the request. Handling body using chunked encoding."); + delegate = new ChunkedInputStream(pushbackInputStream, chunkedBufferSize); + if (instrumenter != null) { + instrumenter.chunkedRequest(); + } + } else { + logger.trace("Client indicated it was sending an entity-body in the request. Handling body using Content-Length header {}.", contentLength); + delegate = new FixedLengthInputStream(pushbackInputStream, contentLength); } - } else if (contentLength != null) { - logger.trace("Client indicated it was sending an entity-body in the request. Handling body using Content-Length header {}.", contentLength); - delegate = new FixedLengthInputStream(pushbackInputStream, contentLength); - } else { - logger.trace("Client indicated it was NOT sending an entity-body in the request"); - } - // Now that we have the InputStream set up to read the body, handle decompression. - if (hasBody) { + // Now that we have the InputStream set up to read the body, handle decompression. // The request may contain more than one value, apply in reverse order. // - These are both using the default 512 buffer size. for (String contentEncoding : request.getContentEncodings().reversed()) { @@ -190,15 +186,23 @@ private void initialize() throws IOException { delegate = new GZIPInputStream(delegate); } } - } - // If we have a fixed length request that is reporting a contentLength larger than the configured maximum, fail earl. - // - Do this last so if anyone downstream wants to read from the InputStream it would work. - // - Note that it is possible that the body is compressed which would mean the contentLength represents the compressed value. - // But when we decompress the bytes the result will be larger than the reported contentLength, so we can safely throw this exception. - if (contentLength != null && maximumContentLength != -1 && contentLength > maximumContentLength) { - String detailedMessage = "The maximum request size has been exceeded. The reported Content-Length is [" + contentLength + "] and the maximum request size is [" + maximumContentLength + "] bytes."; - throw new ContentTooLargeException(maximumContentLength, detailedMessage); + // If we have a fixed length request that is reporting a contentLength larger than the configured maximum, fail early. + // - Do this last so if anyone downstream wants to read from the InputStream it would work. + // - Note that it is possible that the body is compressed which would mean the contentLength represents the compressed value. + // But when we decompress the bytes the result will be larger than the reported contentLength, so we can safely throw this exception. + if (contentLength != null && maximumContentLength != -1 && contentLength > maximumContentLength) { + String detailedMessage = "The maximum request size has been exceeded. The reported Content-Length is [" + contentLength + "] and the maximum request size is [" + maximumContentLength + "] bytes."; + throw new ContentTooLargeException(maximumContentLength, detailedMessage); + } + } else { + // This means that we did not find Content-Length or Transfer-Encoding on the request. Do not attempt to read from the InputStream. + // - Note that the spec indicates it is plausible for a client to send an entity body and omit these two headers and the server can optionally + // read bytes until the end of the InputStream is reached. This would assume Connection: close was also sent because if we do not know + // how to delimit the request we cannot use a persistent connection. + // - We aren't doing any of that - if the client wants to send bytes, it needs to send a Content-Length header, or specify Transfer-Encoding: chunked. + logger.trace("Client indicated it was NOT sending an entity-body in the request"); + delegate = InputStream.nullInputStream(); } } } diff --git a/src/test/java/io/fusionauth/http/BaseSocketTest.java b/src/test/java/io/fusionauth/http/BaseSocketTest.java index 0109f6d0..1a579005 100644 --- a/src/test/java/io/fusionauth/http/BaseSocketTest.java +++ b/src/test/java/io/fusionauth/http/BaseSocketTest.java @@ -67,7 +67,7 @@ private void assertResponse(String request, String chunkedExtension, String resp if (request.contains("Transfer-Encoding: chunked")) { // Chunk in 100 byte increments. Using a smaller chunk size to ensure we don't end up with a single chunk. - body = new String(chunkEncoded(body.getBytes(StandardCharsets.UTF_8), 100, chunkedExtension)); + body = new String(chunkEncode(body.getBytes(StandardCharsets.UTF_8), 100, chunkedExtension)); } request = request.replace("{body}", body); diff --git a/src/test/java/io/fusionauth/http/BaseTest.java b/src/test/java/io/fusionauth/http/BaseTest.java index 7caad08c..ec0ad84a 100644 --- a/src/test/java/io/fusionauth/http/BaseTest.java +++ b/src/test/java/io/fusionauth/http/BaseTest.java @@ -105,21 +105,6 @@ * @author Brian Pontarelli */ public abstract class BaseTest { - protected byte[] compressUsingContentEncoding(byte[] bytes, String contentEncoding) throws Exception { - if (!contentEncoding.isEmpty()) { - var requestEncodings = contentEncoding.toLowerCase().trim().split(","); - for (String part : requestEncodings) { - String encoding = part.trim(); - if (encoding.equals(ContentEncodings.Deflate)) { - bytes = deflate(bytes); - } else if (encoding.equals(ContentEncodings.Gzip)) { - bytes = gzip(bytes); - } - } - } - - return bytes; - } /** * This timeout is used for the HttpClient during each test. If you are in a debugger, you will need to change this timeout to be much * larger, otherwise, the client might truncate the request to the server. @@ -266,11 +251,11 @@ public Object[][] connections() { @DataProvider(name = "contentEncoding") public Object[][] contentEncoding() { return new Object[][]{ - {""}, - {"gzip"}, - {"deflate"}, - {"gzip, deflate"}, - {"deflate, gzip"} + {""}, // No compression + {"gzip"}, // gzip only + {"deflate"}, // deflate only + {"gzip, deflate"}, // gzip, then deflate + {"deflate, gzip"} // deflate, then gzip }; } @@ -460,7 +445,7 @@ protected void assertHTTPResponseEquals(Socket socket, String expectedResponse) } } - protected byte[] chunkEncoded(byte[] bytes, int chunkSize, String chunkedExtension) throws IOException { + protected byte[] chunkEncode(byte[] bytes, int chunkSize, String chunkedExtension) throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); for (var i = 0; i < bytes.length; i += chunkSize) { var endIndex = Math.min(i + chunkSize, bytes.length); @@ -485,6 +470,22 @@ protected byte[] chunkEncoded(byte[] bytes, int chunkSize, String chunkedExtensi return out.toByteArray(); } + protected byte[] compressUsingContentEncoding(byte[] bytes, String contentEncoding) throws Exception { + if (!contentEncoding.isEmpty()) { + var requestEncodings = contentEncoding.toLowerCase().trim().split(","); + for (String part : requestEncodings) { + String encoding = part.trim(); + if (encoding.equals(ContentEncodings.Deflate)) { + bytes = deflate(bytes); + } else if (encoding.equals(ContentEncodings.Gzip)) { + bytes = gzip(bytes); + } + } + } + + return bytes; + } + protected byte[] deflate(byte[] bytes) throws Exception { ByteArrayOutputStream baseOutputStream = new ByteArrayOutputStream(); try (DeflaterOutputStream out = new DeflaterOutputStream(baseOutputStream, true)) { diff --git a/src/test/java/io/fusionauth/http/FormDataTest.java b/src/test/java/io/fusionauth/http/FormDataTest.java index bc9909f5..88f21e27 100644 --- a/src/test/java/io/fusionauth/http/FormDataTest.java +++ b/src/test/java/io/fusionauth/http/FormDataTest.java @@ -286,7 +286,7 @@ public Builder expectResponse(String response) throws Exception { // Convert body to chunked // - Using a small chunk to ensure we end up with more than one chunk. - body = new String(chunkEncoded(body.getBytes(StandardCharsets.UTF_8), 100, null)); + body = new String(chunkEncode(body.getBytes(StandardCharsets.UTF_8), 100, null)); } else { var contentLength = body.getBytes(StandardCharsets.UTF_8).length; if (contentLength > 0) { diff --git a/src/test/java/io/fusionauth/http/io/HTTPInputStreamTest.java b/src/test/java/io/fusionauth/http/io/HTTPInputStreamTest.java index 0125e6b9..5758b8e2 100644 --- a/src/test/java/io/fusionauth/http/io/HTTPInputStreamTest.java +++ b/src/test/java/io/fusionauth/http/io/HTTPInputStreamTest.java @@ -20,12 +20,10 @@ import java.nio.charset.StandardCharsets; import io.fusionauth.http.BaseTest; -import io.fusionauth.http.HTTPValues.ContentEncodings; import io.fusionauth.http.HTTPValues.Headers; import io.fusionauth.http.server.HTTPRequest; import io.fusionauth.http.server.HTTPServerConfiguration; import io.fusionauth.http.server.io.HTTPInputStream; -import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import static org.testng.Assert.assertEquals; @@ -37,6 +35,7 @@ public class HTTPInputStreamTest extends BaseTest { public void read_chunked_withPushback(String contentEncoding) throws Exception { // Ensure that when we read a chunked encoded body that the InputStream returns the correct number of bytes read even when // we read past the end of the current request and use the PushbackInputStream. + // - Test with optional compression as well. String content = "These pretzels are making me thirsty. These pretzels are making me thirsty. These pretzels are making me thirsty."; byte[] payload = content.getBytes(StandardCharsets.UTF_8); @@ -46,26 +45,29 @@ public void read_chunked_withPushback(String contentEncoding) throws Exception { payload = compressUsingContentEncoding(payload, contentEncoding); // Chunk the content, add part of the next request - payload = chunkEncoded(payload, 38, null); + payload = chunkEncode(payload, 38, null); ByteArrayOutputStream out = new ByteArrayOutputStream(); out.write(payload); // Add part of the next request - out.write("GET / HTTP/1.1\r\n".getBytes(StandardCharsets.UTF_8)); + String nextRequest = "GET / HTTP/1.1\n\r"; + byte[] nextRequestBytes = nextRequest.getBytes(StandardCharsets.UTF_8); + out.write(nextRequestBytes); HTTPRequest request = new HTTPRequest(); request.setHeader(Headers.ContentEncoding, contentEncoding); request.setHeader(Headers.TransferEncoding, "chunked"); byte[] bytes = out.toByteArray(); - assertReadWithPushback(bytes, content, contentLength, 16, request); + assertReadWithPushback(bytes, content, contentLength, nextRequestBytes, request); } @Test(dataProvider = "contentEncoding") public void read_fixedLength_withPushback(String contentEncoding) throws Exception { // Ensure that when we read a fixed length body that the InputStream returns the correct number of bytes read even when // we read past the end of the current request and use the PushbackInputStream. + // - Test with optional compression as well. // Fixed length body with the start of the next request in the buffer String content = "These pretzels are making me thirsty. These pretzels are making me thirsty. These pretzels are making me thirsty."; @@ -82,7 +84,9 @@ public void read_fixedLength_withPushback(String contentEncoding) throws Excepti out.write(payload); // Add part of the next request - out.write("GET / HTTP/1.1\r\n".getBytes(StandardCharsets.UTF_8)); + String nextRequest = "GET / HTTP/1.1\n\r"; + byte[] nextRequestBytes = nextRequest.getBytes(StandardCharsets.UTF_8); + out.write(nextRequestBytes); HTTPRequest request = new HTTPRequest(); request.setHeader(Headers.ContentEncoding, contentEncoding); @@ -91,10 +95,10 @@ public void read_fixedLength_withPushback(String contentEncoding) throws Excepti // body length is 113, when compressed it is 68 (gzip) or 78 (deflate) // The number of bytes available is 129. byte[] bytes = out.toByteArray(); - assertReadWithPushback(bytes, content, contentLength, 16, request); + assertReadWithPushback(bytes, content, contentLength, nextRequestBytes, request); } - private void assertReadWithPushback(byte[] bytes, String content, int contentLength, int pushedBack, HTTPRequest request) + private void assertReadWithPushback(byte[] bytes, String content, int contentLength, byte[] pushedBackBytes, HTTPRequest request) throws Exception { int bytesAvailable = bytes.length; HTTPServerConfiguration configuration = new HTTPServerConfiguration().withRequestBufferSize(bytesAvailable + 100); @@ -106,7 +110,9 @@ private void assertReadWithPushback(byte[] bytes, String content, int contentLen byte[] buffer = new byte[configuration.getRequestBufferSize()]; int read = httpInputStream.read(buffer); - // TODO : Hmm.. with compression, this is returning the compressed bytes read instead of the un-compressed bytes read. WTF? + // Note that the HTTPInputStream read will return the number of uncompressed bytes read. So the contentLength passed in + // needs to represent the actual length of the request body, not the value of the Content-Length sent on the request which + // would represent the size of the compressed entity body. assertEquals(read, contentLength); assertEquals(new String(buffer, 0, read), content); @@ -115,12 +121,12 @@ private void assertReadWithPushback(byte[] bytes, String content, int contentLen assertEquals(secondRead, -1); // We have 16 bytes left over - assertEquals(pushbackInputStream.getAvailableBufferedBytesRemaining(), pushedBack); + assertEquals(pushbackInputStream.getAvailableBufferedBytesRemaining(), pushedBackBytes.length); // Next read should start at the next request byte[] leftOverBuffer = new byte[100]; int leftOverRead = pushbackInputStream.read(leftOverBuffer); - assertEquals(leftOverRead, pushedBack); - assertEquals(new String(leftOverBuffer, 0, leftOverRead), "GET / HTTP/1.1\r\n"); + assertEquals(leftOverRead, pushedBackBytes.length); + assertEquals(new String(leftOverBuffer, 0, leftOverRead), new String(pushedBackBytes)); } } From 22cf7d8b2af39ffbb9ff249f2727f7a17f572053 Mon Sep 17 00:00:00 2001 From: Daniel DeGroff Date: Sat, 1 Nov 2025 12:15:43 -0600 Subject: [PATCH 14/21] Cleanup --- .../io/fusionauth/http/MultipartTest.java | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/test/java/io/fusionauth/http/MultipartTest.java b/src/test/java/io/fusionauth/http/MultipartTest.java index 3ecc7fff..413d1a02 100644 --- a/src/test/java/io/fusionauth/http/MultipartTest.java +++ b/src/test/java/io/fusionauth/http/MultipartTest.java @@ -39,7 +39,6 @@ import io.fusionauth.http.server.HTTPServerConfiguration; import org.testng.annotations.Test; import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; /** @@ -128,7 +127,7 @@ public void post_server_configuration_fileTooBig(String scheme) throws Exception content-length: 0\r \r """) - .expectExceptionOnWrite(SocketException.class); + .assertOptionalExceptionOnWrite(SocketException.class); } @Test(dataProvider = "schemes") @@ -187,7 +186,7 @@ public void post_server_configuration_file_upload_reject(String scheme) throws E """) // If the request is large enough, because we throw an exception before we have emptied the InputStream // we will take an exception while trying to write all the bytes to the server. - .expectExceptionOnWrite(SocketException.class); + .assertOptionalExceptionOnWrite(SocketException.class); } @@ -212,7 +211,7 @@ public void post_server_configuration_requestTooBig(String scheme) throws Except """) // If the request is large enough, because we throw an exception before we have emptied the InputStream // we will take an exception while trying to write all the bytes to the server. - .expectExceptionOnWrite(SocketException.class); + .assertOptionalExceptionOnWrite(SocketException.class); } @Test(dataProvider = "schemes") @@ -240,7 +239,7 @@ public void post_server_configuration_requestTooBig_maxBodySize(String scheme) t """) // If the request is large enough, because we throw an exception before we have emptied the InputStream // we will take an exception while trying to write all the bytes to the server. - .expectExceptionOnWrite(SocketException.class); + .assertOptionalExceptionOnWrite(SocketException.class); } private Builder withConfiguration(Consumer configuration) throws Exception { @@ -269,9 +268,14 @@ public Builder(String scheme) { this.scheme = scheme; } - public Builder expectExceptionOnWrite(Class clazz) { - assertNotNull(thrownOnWrite); - assertEquals(thrownOnWrite.getClass(), clazz); + public Builder assertOptionalExceptionOnWrite(Class clazz) { + // Note that this assertion really depends upon the system the test is run on, the size of the request, and the amount of data that can be cached. + // - So this is an optional assertion - if exception is not null, then we should be able to assert some attributes. + // - With the larger sizes this exception is mostly always thrown when running tests locally, but in GHA, it doesn't always occur. + if (thrownOnWrite != null) { + assertEquals(thrownOnWrite.getClass(), clazz); + } + return this; } From 82a97a82139f1a62fcfbb50b47e1ec0d072e0a97 Mon Sep 17 00:00:00 2001 From: Daniel DeGroff Date: Sat, 1 Nov 2025 12:22:14 -0600 Subject: [PATCH 15/21] Cleanup --- .../io/fusionauth/http/server/HTTPRequest.java | 4 ++-- .../io/fusionauth/http/CompressionTest.java | 18 +++++++++++------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/main/java/io/fusionauth/http/server/HTTPRequest.java b/src/main/java/io/fusionauth/http/server/HTTPRequest.java index 42310d73..f04a5609 100644 --- a/src/main/java/io/fusionauth/http/server/HTTPRequest.java +++ b/src/main/java/io/fusionauth/http/server/HTTPRequest.java @@ -790,8 +790,8 @@ private void decodeHeader(String name, String value) { continue; } - // The HTTP/1.1 standard recommends that the servers supporting gzip also recognize x-gzip as an alias, for compatibility purposes. - if (encoding.equals(ContentEncodings.XGzip)) { + // The HTTP/1.1 standard recommends that the servers supporting gzip also recognize x-gzip as an alias for compatibility. + if (encoding.equalsIgnoreCase(ContentEncodings.XGzip)) { encoding = ContentEncodings.Gzip; } diff --git a/src/test/java/io/fusionauth/http/CompressionTest.java b/src/test/java/io/fusionauth/http/CompressionTest.java index 6808e3f5..a8660132 100644 --- a/src/test/java/io/fusionauth/http/CompressionTest.java +++ b/src/test/java/io/fusionauth/http/CompressionTest.java @@ -40,6 +40,7 @@ import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotEquals; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; import static org.testng.AssertJUnit.fail; @@ -258,14 +259,17 @@ public void compressWithContentLength(String scheme) throws Exception { assertEquals(req.isChunked(), false); - // We forced a Content-Length by telling the JDK client not to chunk, so it will be present. It will - // be used in theory - and it should be ok as long as we are using it to count compressed bytes? - // TODO : Once I add the push back tests for compression, we'll see if this still works. We may - // need to manually remove the Content-Length header when we know the body to be compressed. + // We forced a Content-Length by telling the JDK client not to chunk, so it will be present. + // - We can still correctly use the Content-Length because it is used by the FixedLengthInputStream which + // is below the decompression. See HTTPInputStream.initialize var contentLength = req.getHeader(Headers.ContentLength); assertEquals(contentLength, payload.length + ""); - String body = new String(req.getInputStream().readAllBytes()); + // Because we are compressed, don't expect the final payload to be equal to the contentLength + byte[] readBytes = req.getInputStream().readAllBytes(); + assertNotEquals(readBytes.length, contentLength); + + String body = new String(readBytes); assertEquals(body, bodyString); res.setHeader(Headers.ContentType, "text/plain"); @@ -326,9 +330,9 @@ public Object[][] compressedChunkedSchemes() { {"deflate, gzip", "deflate, gzip"}, {"gzip, deflate", "gzip, deflate"}, - // x-gzip alias, https + // x-gzip with mixed case alias, https {"x-gzip", "deflate, gzip"}, - {"x-gzip, deflate", "gzip, deflate"}, + {"X-Gzip, deflate", "gzip, deflate"}, {"deflate, x-gzip", "gzip, deflate"}, }; From 49895a3c77de4fcdc15b29cd0abd4c3651600f32 Mon Sep 17 00:00:00 2001 From: Daniel DeGroff Date: Sat, 1 Nov 2025 12:28:30 -0600 Subject: [PATCH 16/21] readme, version --- README.md | 17 +++++++---------- build.savant | 2 +- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 9f613c02..bc5886ec 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ### Latest versions -* Latest stable version: `1.3.1` +* Latest stable version: `1.4.0` * Now with 100% more virtual threads! * Prior stable version `0.3.7` @@ -27,20 +27,20 @@ To add this library to your project, you can include this dependency in your Mav io.fusionauth java-http - 1.3.1 + 1.4.0 ``` If you are using Gradle, you can add this to your build file: ```groovy -implementation 'io.fusionauth:java-http:1.3.1' +implementation 'io.fusionauth:java-http:1.4.0' ``` If you are using Savant, you can add this to your build file: ```groovy -dependency(id: "io.fusionauth:java-http:1.3.1") +dependency(id: "io.fusionauth:java-http:1.4.0") ``` ## Examples Usages: @@ -231,16 +231,13 @@ The general requirements and roadmap are as follows: ### Server tasks * [x] Basic HTTP 1.1 -* [x] Support Accept-Encoding (gzip, deflate) +* [x] Support Accept-Encoding (gzip, deflate), by default and per response options. * [x] Support Content-Encoding (gzip, deflate) * [x] Support Keep-Alive * [x] Support Expect-Continue 100 -* [x] Support chunked request -* [x] Support chunked response -* [x] Support streaming entity bodies (via chunking likely) -* [x] Support compression (default and per response options) +* [x] Support Transfer-Encoding: chunked on request for streaming. +* [x] Support Transfer-Encoding: chunked on response * [x] Support cookies in request and response -* [x] Clean up HTTPRequest * [x] Support form data * [x] Support multipart form data * [x] Support TLS diff --git a/build.savant b/build.savant index 06b51d28..42624b38 100644 --- a/build.savant +++ b/build.savant @@ -18,7 +18,7 @@ restifyVersion = "4.2.1" slf4jVersion = "2.0.17" testngVersion = "7.11.0" -project(group: "io.fusionauth", name: "java-http", version: "1.3.1", licenses: ["ApacheV2_0"]) { +project(group: "io.fusionauth", name: "java-http", version: "1.4.0", licenses: ["ApacheV2_0"]) { workflow { fetch { // Dependency resolution order: From b5996187dbbfcc03fb3d786fa03656ebb4433c24 Mon Sep 17 00:00:00 2001 From: Daniel DeGroff Date: Sat, 1 Nov 2025 12:28:53 -0600 Subject: [PATCH 17/21] readme, version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b828014e..a32143b3 100644 --- a/pom.xml +++ b/pom.xml @@ -2,7 +2,7 @@ 4.0.0 io.fusionauth java-http - 1.3.0 + 1.4.0 jar Java HTTP library (client and server) From dc4d6ddf52e95772bf6d9ee647875f25ee092c47 Mon Sep 17 00:00:00 2001 From: Daniel DeGroff Date: Sat, 1 Nov 2025 12:34:55 -0600 Subject: [PATCH 18/21] cleanup --- src/test/java/io/fusionauth/http/BaseTest.java | 1 - src/test/java/io/fusionauth/http/io/HTTPInputStreamTest.java | 4 +--- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/test/java/io/fusionauth/http/BaseTest.java b/src/test/java/io/fusionauth/http/BaseTest.java index ec0ad84a..93eb2924 100644 --- a/src/test/java/io/fusionauth/http/BaseTest.java +++ b/src/test/java/io/fusionauth/http/BaseTest.java @@ -465,7 +465,6 @@ protected byte[] chunkEncode(byte[] bytes, int chunkSize, String chunkedExtensio out.write("\r\n".getBytes(StandardCharsets.UTF_8)); } - out.write(("0\r\n\r\n".getBytes(StandardCharsets.UTF_8))); return out.toByteArray(); } diff --git a/src/test/java/io/fusionauth/http/io/HTTPInputStreamTest.java b/src/test/java/io/fusionauth/http/io/HTTPInputStreamTest.java index 5758b8e2..bf8f4bf7 100644 --- a/src/test/java/io/fusionauth/http/io/HTTPInputStreamTest.java +++ b/src/test/java/io/fusionauth/http/io/HTTPInputStreamTest.java @@ -92,8 +92,6 @@ public void read_fixedLength_withPushback(String contentEncoding) throws Excepti request.setHeader(Headers.ContentEncoding, contentEncoding); request.setHeader(Headers.ContentLength, compressedLength + ""); - // body length is 113, when compressed it is 68 (gzip) or 78 (deflate) - // The number of bytes available is 129. byte[] bytes = out.toByteArray(); assertReadWithPushback(bytes, content, contentLength, nextRequestBytes, request); } @@ -120,7 +118,7 @@ private void assertReadWithPushback(byte[] bytes, String content, int contentLen int secondRead = httpInputStream.read(buffer); assertEquals(secondRead, -1); - // We have 16 bytes left over + // Expect that we have the number of bytes left over we expect assertEquals(pushbackInputStream.getAvailableBufferedBytesRemaining(), pushedBackBytes.length); // Next read should start at the next request From ac8f16b44f4e55e3493188ef47a5389838827fb2 Mon Sep 17 00:00:00 2001 From: Daniel DeGroff Date: Mon, 3 Nov 2025 18:39:52 -0700 Subject: [PATCH 19/21] copy --- src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java b/src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java index ee5f3835..c7d58d4d 100644 --- a/src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java +++ b/src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java @@ -31,7 +31,7 @@ import io.fusionauth.http.server.Instrumenter; /** - * An InputStream intended to be read the HTTP request body. + * An InputStream intended to read the HTTP request body. *

* This will handle fixed length requests, chunked requests as well as decompression if necessary. * From b089176a9d7b11897d57ab8b539b761c6af86ef4 Mon Sep 17 00:00:00 2001 From: Daniel DeGroff Date: Mon, 3 Nov 2025 18:50:31 -0700 Subject: [PATCH 20/21] FusionAuth -> Java in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index bc5886ec..b93622d8 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -## FusionAuth HTTP client and server ![semver 2.0.0 compliant](http://img.shields.io/badge/semver-2.0.0-brightgreen.svg?style=flat-square) [![test](https://github.com/FusionAuth/java-http/actions/workflows/test.yml/badge.svg)](https://github.com/FusionAuth/java-http/actions/workflows/test.yml) +## Java HTTP client and server ![semver 2.0.0 compliant](http://img.shields.io/badge/semver-2.0.0-brightgreen.svg?style=flat-square) [![test](https://github.com/FusionAuth/java-http/actions/workflows/test.yml/badge.svg)](https://github.com/FusionAuth/java-http/actions/workflows/test.yml) ### Latest versions From 2deed2681023fc3caf485b8a786f76567ce03da5 Mon Sep 17 00:00:00 2001 From: Daniel DeGroff Date: Wed, 5 Nov 2025 16:03:58 -0700 Subject: [PATCH 21/21] Remove dead variable --- src/main/java/io/fusionauth/http/server/HTTPRequest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/io/fusionauth/http/server/HTTPRequest.java b/src/main/java/io/fusionauth/http/server/HTTPRequest.java index f04a5609..da5e19e9 100644 --- a/src/main/java/io/fusionauth/http/server/HTTPRequest.java +++ b/src/main/java/io/fusionauth/http/server/HTTPRequest.java @@ -783,7 +783,6 @@ private void decodeHeader(String name, String value) { case Headers.ContentEncodingLower: String[] encodings = value.split(","); List contentEncodings = new ArrayList<>(1); - int encodingIndex = 0; for (String encoding : encodings) { encoding = encoding.trim(); if (encoding.isEmpty()) { @@ -796,7 +795,6 @@ private void decodeHeader(String name, String value) { } contentEncodings.add(encoding); - encodingIndex++; } setContentEncodings(contentEncodings);